git clone functionality added

src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java

@@ -1,28 +1,24 @@
 package com.olympus.apollo.controllers.FeApi;
 
+import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 
+import com.olympus.apollo.dto.*;
+import com.olympus.apollo.services.GitService;
+import org.eclipse.jgit.api.errors.GitAPIException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
 
-import com.olympus.apollo.dto.KSGitInfoDTO;
-import com.olympus.apollo.dto.KSGitUploadDTO;
 import com.olympus.apollo.models.KSGitInfo;
 import com.olympus.apollo.models.KSGitIngestionInfo;
 import com.olympus.apollo.repository.KSGitInfoRepository;
 import com.olympus.apollo.repository.KSGitIngestionInfoRepository;
 import com.olympus.apollo.services.KSGitInfoService;
 
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
-
 @CrossOrigin(origins = "http://localhost:5173")
 @RestController
 @RequestMapping("/fe-api/ks_git_repos")
@@ -33,6 +29,9 @@ public class KSGitController {
 	@Autowired
 	private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
 
+	@Autowired
+	private GitService gitService;
+
 	@GetMapping("")
 	public List<KSGitInfo> listGitInfo() {
 		List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
@@ -75,4 +74,53 @@ public class KSGitController {
 		return ResponseEntity.ok("Upload successful");
 	}
 
+	//clone the master branch from remote repository
+	@PostMapping("/clone")
+	public GitCloneOutput gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitAPIException, IOException {
+		KSGitInfo ksGitInfo = new KSGitInfo();
+		ksGitInfo.setRepoName(gitCloneInput.getRepoName());
+		ksGitInfo.setBranch(gitCloneInput.getBranch());
+		ksGitInfo.setCommitId(gitCloneInput.getCommitId());
+		ksGitInfo.setRepoPath(gitCloneInput.getRepoPath());
+		ksGitInfo.setIngestionStatus("NEW");
+		ksGitInfo.setIngestionDate(new Date());
+		ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
+
+		KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
+		HashMap<String, String> metadata = new HashMap<>();
+
+		metadata.put("KsApplicationName", gitCloneInput.getRepoName());
+		metadata.put("KsDoctype", "gitrepository");
+		metadata.put("KsDocSource", "gitlab");
+		metadata.put("KsFileSource", gitCloneInput.getRepoName());
+
+		metadata.put("KsBranch", gitCloneInput.getBranch());
+		metadata.put("KsRepoName", gitCloneInput.getRepoName());
+
+		ksGitIngestionInfo.setMetadata(metadata);
+		ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
+		ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
+		ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
+		ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
+
+		ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
+		ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
+		ksGitInfoRepository.save(ksGitInfo);
+
+		return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
+	}
+	/*
+	curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \
+	--header 'Content-Type: application/json' \
+	--header 'Authorization: Bearer ' \
+	--data '{"repoName":"shellExecutionThroughAPI","group":"automationtester23","source":"https://github.com","tokenType":"github","branch":"master","commitId":"dummy","repoPath":"C:\\repos\\olympus_ai\\gitClone","minChunkSizeToEmbed":20,"maxNumberOfChunks":1988,"minChunkSize":200,"defaultChunkSize":1988}
+'
+	 */
+
+	//pull latest changes from master branch
+	@GetMapping("/pullchanges/{repoName}")
+	public GitPullOutput gitPull(@PathVariable String repoName){
+		return gitService.pullChanges(repoName);
+	}
+
 }

src/main/java/com/olympus/apollo/controllers/TestController.java

@@ -2,6 +2,7 @@ package com.olympus.apollo.controllers;
 
 import java.util.List;
 
+import com.olympus.apollo.services.GitService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -27,6 +28,9 @@ public class TestController {
 	@Autowired
 	GitRepositoryIngestor gitRepositoryIngestor;
 
+	@Autowired
+	GitService gitService;
+
 	private static final Logger logger = LoggerFactory.getLogger(TestController.class);
 
 	@GetMapping("test/ingestion_loop")
@@ -64,6 +68,25 @@ public class TestController {
 		}
 	}
 
+	@GetMapping("test/reingest_repo/{repoName}")
+	public ResponseEntity<String> ReIngestRepo(@PathVariable String repoName) {
+		try {
+			gitService.pullChanges(repoName);
+			gitRepositoryIngestor.ReIngestGitRepository(repoName);
+			return ResponseEntity.ok("Ingestion Started");
+		} catch (Exception e) {
+
+			logger.error("Error during ingestion start", e);
+
+			return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
+					.body("Error starting ingestion: " + e.getMessage());
+		}
+	}
+	/*
+	curl --location 'http://localhost:8082/test/reingest_repo/shellExecutionThroughAPI' \
+	--header 'Authorization: Bearer '
+	 */
+
 	@GetMapping("test/check_ingestion_status/{repoName}")
 	public ResponseEntity<IngestionOutput> checkIngestionStatus(@PathVariable String repoName) {
 		try {
@@ -79,5 +102,4 @@ public class TestController {
 			return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorOutput);
 		}
 	}
-
 }

src/main/java/com/olympus/apollo/dto/GitCloneInput.java

@@ -0,0 +1,23 @@
+package com.olympus.apollo.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.HashMap;
+
+@Setter
+@Getter
+public class GitCloneInput {
+    private String source;
+    private String repoName;
+    private String group;
+    private String tokenType;
+
+    private String branch;
+    private String commitId;
+    private String repoPath;
+    private int minChunkSizeToEmbed;
+    private int maxNumberOfChunks;
+    private int minChunkSize;
+    private int defaultChunkSize;
+}

src/main/java/com/olympus/apollo/dto/GitCloneOutput.java

@@ -0,0 +1,10 @@
+package com.olympus.apollo.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Setter @Getter
+public class GitCloneOutput {
+    private String message;
+    private String repoName;
+}

src/main/java/com/olympus/apollo/dto/GitPullOutput.java

@@ -0,0 +1,13 @@
+package com.olympus.apollo.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Map;
+
+@Setter @Getter
+public class GitPullOutput {
+    private Map<String,String > changes;
+    private String repoName;
+    private String message;
+}

src/main/java/com/olympus/apollo/models/KSGitInfo.java

@@ -1,6 +1,7 @@
 package com.olympus.apollo.models;
 
 import java.util.Date;
+import java.util.HashMap;
 
 import org.springframework.data.annotation.Id;
 import org.springframework.data.mongodb.core.mapping.Document;
@@ -20,6 +21,7 @@ public class KSGitInfo {
     private String commitId;
     private String repoPath;
     private KSGitIngestionInfo ksGitIngestionInfo;
+    private HashMap<String, String> gitModifiedFiles;
     private String ingestionStatus;
     private Date ingestionDate;
     private String ingestionDateFormat;

src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java

@@ -1,8 +1,11 @@
 package com.olympus.apollo.repository;
 
+import java.util.List;
 import java.util.Optional;
 
+import com.olympus.apollo.models.VectorStore;
 import org.springframework.data.mongodb.repository.MongoRepository;
+import org.springframework.data.mongodb.repository.Query;
 import org.springframework.data.rest.core.annotation.RepositoryRestResource;
 import org.springframework.stereotype.Repository;
 import org.springframework.web.bind.annotation.CrossOrigin;
@@ -12,4 +15,7 @@ import com.olympus.apollo.models.KSGitInfo;
 @Repository
 public interface KSGitInfoRepository extends MongoRepository<KSGitInfo, String> {
 	Optional<KSGitInfo> findByRepoName(String repoName);
+
+	@Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsApplicationName': ?1}")
+	Optional<KSGitInfo> findByMetadataAndRepoName(String repoName, String KsApplicationName);
 }

src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java

@@ -8,6 +8,7 @@ import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
+import java.util.Optional;
 
 
 @Repository
@@ -24,4 +25,8 @@ public interface VectorStoreRepository extends MongoRepository<VectorStore, Stri
 
     @Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}")
     List<VectorStore> findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
+
+    @Query("{'metadata.filePath': ?0}")
+    Optional<VectorStore> findByFilePath(String filePath);
+
 }

src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java

@@ -1,6 +1,7 @@
 package com.olympus.apollo.services;
 
 import java.io.File;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Date;
@@ -12,7 +13,9 @@ import java.util.concurrent.CompletableFuture;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.olympus.apollo.repository.VectorStoreRepository;
 import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.treewalk.TreeWalk;
@@ -36,9 +39,15 @@ public class GitRepositoryIngestor {
 
     private final VectorStore vectorStore;
 
+    @Value("${gitlab.path}")
+    private String localRepoPath;
+
     @Autowired
     private KSGitInfoRepository ksGitInfoRepository;
 
+    @Autowired
+    private VectorStoreRepository vectorStoreRepository;
+
     public GitRepositoryIngestor(VectorStore vectorStore) {
         this.vectorStore = vectorStore;
     }
@@ -67,7 +76,8 @@ public class GitRepositoryIngestor {
 
 
     private void ingestRepo(String repo, KSGitInfo ksGitInfo) {
-		String repoPath = "/mnt/apollo_storage/repository" +"\\"+ repo + "\\";
+        String repoPath = localRepoPath+"/"+ repo + "/";
+        //String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\";
         logger.info("Repository path : " + repoPath);
         try (Git git = Git.open(new File(repoPath))) {
             ksGitInfo.setIngestionStatus("IN PROGRESS");
@@ -125,6 +135,141 @@ public class GitRepositoryIngestor {
         }
     }
 
+    public CompletableFuture<Void> ReIngestGitRepository(String repo) throws GitAPIException, IOException {
+        //String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
+        //String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
+        //logger.info("Repository path : " + repoPath);
+
+        Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
+        if (optionalDocument.isPresent()) {
+            KSGitInfo ksGitInfo = optionalDocument.get();
+            if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
+                reIngestRepo(repo, ksGitInfo);
+            } else {
+                logger.info("OOPS: Document is already Injected");
+            }
+        } else {
+            logger.info("OOPS: Document Not found");
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    private void reIngestRepo(String repo, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
+
+
+        HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
+
+
+        List<String> filePathsToDelete = new ArrayList<>();
+        List<String> filePathsToEmbed = new ArrayList<>();
+
+        for (Map.Entry<String, String> entry : modifiedFiles.entrySet()) {
+            switch (entry.getKey()) {
+                case "MODIFY":
+                    String[] modifiedFileList = entry.getValue().split(",");
+                    for (String modifiedFile : modifiedFileList) {
+                        filePathsToDelete.add(modifiedFile);
+                        filePathsToEmbed.add(modifiedFile);
+                    }
+                    break;
+                case "ADD":
+                    String[] addedFileList = entry.getValue().split(",");
+                    for (String addFile : addedFileList) {
+                        filePathsToEmbed.add(addFile);
+                    }
+                    break;
+                case "DELETE":
+                    String[] deletedFileList = entry.getValue().split(",");
+                    for (String deletedFile : deletedFileList) {
+                        filePathsToDelete.add(deletedFile);
+                    }
+                    break;
+                default:
+                    break;
+            }
+            for (String fileToDelete : filePathsToDelete) {
+                Optional<com.olympus.apollo.models.VectorStore> optionalDocument = vectorStoreRepository.findByFilePath(fileToDelete);
+                if (optionalDocument.isPresent()) {
+                    String vectorStoreId = optionalDocument.get().getId();
+                    vectorStoreRepository.deleteById(vectorStoreId);
+                }
+            }
+
+        }
+        String repoPath = localRepoPath +"/"+ repo + "/";
+        //String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\"; //need to modify before deploy
+        logger.info("Repository path : " + repoPath);
+
+        try (Git git = Git.open(new File(repoPath))) {
+            ksGitInfo.setIngestionStatus("IN PROGRESS");
+            KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
+            logger.info("Metadata : " + ingestionInfo.getMetadata());
+            ksGitInfoRepository.save(ksGitInfo);
+
+            Repository repository = git.getRepository();
+            RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
+
+                try (TreeWalk treeWalk = new TreeWalk(repository)) {
+                    treeWalk.addTree(latestCommit.getTree());
+                    treeWalk.setRecursive(true);
+                List<Document> documents = new ArrayList<>();
+                    for (String filePath : filePathsToEmbed) {
+
+                        String[] parts = filePath.split("/");
+                        String fileName = parts[parts.length - 1];
+                        if (isRelevantFile(fileName)) {
+                            boolean fileFound = false;
+                            while (treeWalk.next()) {
+                                if (treeWalk.getPathString().equals(filePath)) {
+                                    fileFound = true;
+                                    byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
+                                    String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
+
+                                    Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
+                                    metadata.put("filePath", filePath);
+                                    metadata.put("fileName", fileName);
+
+                                    Document doc = new Document(fileContentStr);
+                                    doc.getMetadata().putAll(metadata);
+
+                                    doc.getMetadata().putAll(ingestionInfo.getMetadata());
+                                    documents.add(doc);
+                                    break;
+                                }
+                            }
+                            if (!fileFound) {
+                                logger.warn("File not found in repository: " + filePath);
+                            }
+                            // Reset TreeWalk to start from the beginning for the next file
+                            treeWalk.reset();
+                            treeWalk.addTree(latestCommit.getTree());
+                            treeWalk.setRecursive(true);
+                        }
+                    }
+
+
+                TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
+                        ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
+                        ingestionInfo.getMaxNumberOfChunks(), false);
+
+                List<Document> splitDocuments = splitter.split(documents);
+                logger.info("Number of documents: " + splitDocuments.size());
+                vectorStore.add(splitDocuments);
+                logger.info("Documents embedded");
+                }
+
+
+                ksGitInfo.setIngestionStatus("INGESTED");
+                ksGitInfo.setIngestionDate(new Date());
+                ksGitInfoRepository.save(ksGitInfo);
+
+        } catch (Exception e) {
+                ksGitInfo.setIngestionStatus("ERROR");
+                ksGitInfoRepository.save(ksGitInfo);
+                logger.error("Error during ingestion", e);
+            }
+        }
+
     public IngestionOutput checkIngestionStatus(String repoName) {
         Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repoName);
         IngestionOutput ingestionOutput = new IngestionOutput();
@@ -137,7 +282,7 @@ public class GitRepositoryIngestor {
             }
         } else {
             ingestionOutput.setStatus("ERROR");
-	        ingestionOutput.setMessage("Document Not Found");
+            ingestionOutput.setMessage("Repository Not Found");
         }
         return ingestionOutput;
     }

src/main/java/com/olympus/apollo/services/GitService.java

@@ -0,0 +1,204 @@
+package com.olympus.apollo.services;
+
+import com.olympus.apollo.dto.GitCloneOutput;
+import com.olympus.apollo.dto.GitPullOutput;
+import com.olympus.apollo.models.KSGitInfo;
+import com.olympus.apollo.repository.KSGitInfoRepository;
+import com.olympus.apollo.repository.VectorStoreRepository;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.RefNotFoundException;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.diff.DiffFormatter;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.eclipse.jgit.treewalk.AbstractTreeIterator;
+import org.eclipse.jgit.treewalk.CanonicalTreeParser;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+@Service
+public class GitService {
+
+    @Autowired
+    private KSGitInfoRepository ksGitInfoRepository;
+
+    @Value("${gitlab.path}")
+    private String localPath;
+
+    @Value("${gitlab.cloud.token}")
+    private String cloudGitlabToken;
+
+    @Value("${gitlab.onpremises.token}")
+    private String onPremisesGitlabToken;
+
+    private String gitHubToken="23";
+
+    public GitCloneOutput cloneRepository(String Source,String repoName,String group, String tokenType) throws GitAPIException {
+        GitCloneOutput gitCloneOutput= new GitCloneOutput();
+        String gitlabToken;
+        String remoteRepoUrl;
+        switch (tokenType){
+            case "cloud":
+                gitlabToken=cloudGitlabToken;
+                remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
+                break;
+            case "onpremises":
+                gitlabToken=onPremisesGitlabToken;
+                remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
+                break;
+            case "github":
+                gitlabToken=gitHubToken;
+                remoteRepoUrl= Source+"/"+group+"/"+repoName+".git";
+                //String remoteRepoUrl= "https://github.com/automationtester23/shellExecutionThroughAPI.git";
+                break;
+            default:
+                gitCloneOutput.setMessage("tokenType is invalid");
+                gitCloneOutput.setRepoName(repoName);
+                return gitCloneOutput;
+        }
+        System.out.println("remoteUrl : "+remoteRepoUrl);
+        System.out.println("gitlab token : "+gitlabToken);
+
+        File cloneDirectory = new File(localPath,repoName);
+        System.out.println("cloneDirectory : "+cloneDirectory);
+        if(tokenType.equals("github")){
+            Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
+        }else {
+            Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",gitlabToken)).call();
+        }
+
+        checkOutRepository(repoName,"master");
+
+        gitCloneOutput.setRepoName(repoName);
+        gitCloneOutput.setMessage(repoName +" cloned successfully");
+
+        return gitCloneOutput;
+    }
+
+    public String checkOutRepository(String repoName, String branchName) {
+        File repoDirectory = new File(localPath,repoName);
+        try(Git git = Git.open(repoDirectory)) {
+            git.checkout().setName(branchName).setCreateBranch(false).call();
+            return branchName+" checkout successful";
+        } catch (RefNotFoundException  e) {
+            try(Git git = Git.open(repoDirectory)) {
+                git.checkout().setName("origin/"+branchName).setCreateBranch(false).call();
+                return branchName+" checkout successful";
+            } catch (RefNotFoundException  ex) {
+                return "checkout failed with error "+e.getMessage();
+            }catch (Exception ex2){
+                System.out.println("An error occurred: "+ ex2.getMessage());
+                return "checkout failed with error "+ ex2.getMessage();
+            }
+        } catch (Exception e){
+            System.out.println("An error occurred: "+ e.getMessage());
+            return "checkout failed with error "+e.getMessage();
+        }
+    }
+
+    public GitPullOutput pullChanges(String repoName) {
+        File repoDirectory = new File(localPath,repoName);
+        GitPullOutput gitPullOutput =new GitPullOutput();
+        Map<String,String> gitdiff=null;
+        try(Git git = Git.open(repoDirectory)) {
+
+            String message = checkOutRepository(repoName,"master");
+            System.out.println("message from checkoutRepository : "+message);
+            gitdiff = getModifiedFiles(git);
+
+            HashMap<String,String> modifiedFiles = new HashMap<>();
+
+            for (Map.Entry<String,String> entry: gitdiff.entrySet()){
+                System.out.println(entry.getKey()+" "+entry.getValue());
+                modifiedFiles.put(entry.getKey(),entry.getValue());
+            }
+
+            Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByMetadataAndRepoName(repoName,repoName);
+            if (optionalDocument.isPresent()) {
+                KSGitInfo ksGitInfo = optionalDocument.get();
+                ksGitInfo.setGitModifiedFiles(modifiedFiles);
+                ksGitInfoRepository.save(ksGitInfo);
+            }else {
+                System.out.println("Records not present");
+            }
+
+            git.pull().call();
+            /*
+            if (commitId != null && !commitId.isEmpty()) {
+                String message1 = checkOutRepository(repoName,commitId);
+                System.out.println("message1 : "+message1);
+            }*/
+            gitPullOutput.setRepoName(repoName);
+            gitPullOutput.setMessage("repo pulled");
+            gitPullOutput.setChanges(gitdiff);
+        } catch (RefNotFoundException  e) {
+            System.out.println("checkout failed with error "+e.getMessage());
+            gitPullOutput.setRepoName(repoName);
+            gitPullOutput.setMessage("error occurred "+e.getMessage());
+            gitPullOutput.setChanges(gitdiff);
+        } catch (Exception e){
+            System.out.println("An error occurred: "+ e.getMessage());
+            gitPullOutput.setRepoName(repoName);
+            gitPullOutput.setMessage("error occurred "+e.getMessage());
+            gitPullOutput.setChanges(gitdiff);
+        }
+        return gitPullOutput;
+    }
+
+    private Map<String,String> getModifiedFiles(Git git) throws GitAPIException, IOException {
+        Repository repository = git.getRepository();
+        git.fetch().call();
+
+        AbstractTreeIterator oldTreeParser = prepareTreeParser(repository, "HEAD");
+        AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/master");
+
+        Map<String, String> modifiedFiles = new HashMap<>();
+        try (DiffFormatter formatter = new DiffFormatter(System.out)) {
+            formatter.setRepository(repository);
+            List<DiffEntry> entries = formatter.scan(oldTreeParser, newTreeParser);
+
+            for (DiffEntry entry : entries) {
+                String changeType = entry.getChangeType().name();
+                String filePath = entry.getNewPath();
+                if(modifiedFiles.containsKey(changeType)){
+                    String existingFilePaths = modifiedFiles.get(changeType);
+                    modifiedFiles.put(changeType,existingFilePaths+","+filePath);
+                }else {
+                    modifiedFiles.put(changeType,filePath);
+                }
+            }
+        }
+
+        return modifiedFiles;
+
+    }
+
+    private AbstractTreeIterator prepareTreeParser(Repository repository, String ref) throws IOException, IOException {
+        ObjectId id = repository.resolve(ref);
+        try (var reader = repository.newObjectReader()) {
+            return new CanonicalTreeParser(null, reader, repository.resolve(id.name() + "^{tree}"));
+        }
+    }
+
+    // Method to extract the protocol part
+    private static String getProtocol(String url) {
+        int protocolEndIndex = url.indexOf("://");
+        return protocolEndIndex != -1 ? url.substring(0, protocolEndIndex + 3) : "";
+    }
+
+    // Method to extract the domain part
+    private static String getDomain(String url) {
+        int protocolEndIndex = url.indexOf("://");
+        return protocolEndIndex != -1 ? url.substring(protocolEndIndex + 3) : url;
+    }
+}

src/main/resources/application.properties

@@ -29,3 +29,7 @@ spring.servlet.multipart.max-request-size=500000MB
 
 #path to the repository
 ingestion.repository.basepath=C:\\Users\\andrea.terzani\\dev\\Olympus
+
+gitlab.token=
+#gitlab.path=C:\\repos\\olympus_ai\\gitClone
+gitlab.path=/mnt/apollo_storage/repository;