admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

git clone functionality added

Browse Source
This commit is contained in:
sumedh
2024-08-27 17:25:22 +05:30
parent 8c40c31078
commit 35bd7e3cf5
11 changed files with 787 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java
View File

@@ -1,28 +1,24 @@
package com.olympus.apollo.controllers.FeApi;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import com.olympus.apollo.dto.*;
import com.olympus.apollo.services.GitService;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import com.olympus.apollo.dto.KSGitInfoDTO;
import com.olympus.apollo.dto.KSGitUploadDTO;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.KSGitIngestionInfoRepository;
import com.olympus.apollo.services.KSGitInfoService;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
@CrossOrigin(origins = "http://localhost:5173")
@RestController
@RequestMapping("/fe-api/ks_git_repos")
@@ -33,6 +29,9 @@ public class KSGitController {
@Autowired
private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
@Autowired
private GitService gitService;
@GetMapping("")
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
@@ -75,4 +74,53 @@ public class KSGitController {
return ResponseEntity.ok("Upload successful");
}
//clone the master branch from remote repository
@PostMapping("/clone")
public GitCloneOutput gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitAPIException, IOException {
KSGitInfo ksGitInfo = new KSGitInfo();
ksGitInfo.setRepoName(gitCloneInput.getRepoName());
ksGitInfo.setBranch(gitCloneInput.getBranch());
ksGitInfo.setCommitId(gitCloneInput.getCommitId());
ksGitInfo.setRepoPath(gitCloneInput.getRepoPath());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", gitCloneInput.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", gitCloneInput.getRepoName());
metadata.put("KsBranch", gitCloneInput.getBranch());
metadata.put("KsRepoName", gitCloneInput.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
}
/*
curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer ' \
--data '{"repoName":"shellExecutionThroughAPI","group":"automationtester23","source":"https://github.com","tokenType":"github","branch":"master","commitId":"dummy","repoPath":"C:\\repos\\olympus_ai\\gitClone","minChunkSizeToEmbed":20,"maxNumberOfChunks":1988,"minChunkSize":200,"defaultChunkSize":1988}
'
*/
//pull latest changes from master branch
@GetMapping("/pullchanges/{repoName}")
public GitPullOutput gitPull(@PathVariable String repoName){
return gitService.pullChanges(repoName);
}
}

24
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -2,6 +2,7 @@ package com.olympus.apollo.controllers;
import java.util.List;
import com.olympus.apollo.services.GitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -27,6 +28,9 @@ public class TestController {
@Autowired
GitRepositoryIngestor gitRepositoryIngestor;
@Autowired
GitService gitService;
private static final Logger logger = LoggerFactory.getLogger(TestController.class);
@GetMapping("test/ingestion_loop")
@@ -64,6 +68,25 @@ public class TestController {
}
}
@GetMapping("test/reingest_repo/{repoName}")
public ResponseEntity<String> ReIngestRepo(@PathVariable String repoName) {
try {
gitService.pullChanges(repoName);
gitRepositoryIngestor.ReIngestGitRepository(repoName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
logger.error("Error during ingestion start", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body("Error starting ingestion: " + e.getMessage());
}
}
/*
curl --location 'http://localhost:8082/test/reingest_repo/shellExecutionThroughAPI' \
--header 'Authorization: Bearer '
*/
@GetMapping("test/check_ingestion_status/{repoName}")
public ResponseEntity<IngestionOutput> checkIngestionStatus(@PathVariable String repoName) {
try {
@@ -79,5 +102,4 @@ public class TestController {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorOutput);
}
}
}

23
src/main/java/com/olympus/apollo/dto/GitCloneInput.java Normal file
View File

@@ -0,0 +1,23 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.HashMap;
@Setter
@Getter
public class GitCloneInput {
private String source;
private String repoName;
private String group;
private String tokenType;
private String branch;
private String commitId;
private String repoPath;
private int minChunkSizeToEmbed;
private int maxNumberOfChunks;
private int minChunkSize;
private int defaultChunkSize;
}

10
src/main/java/com/olympus/apollo/dto/GitCloneOutput.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Setter @Getter
public class GitCloneOutput {
private String message;
private String repoName;
}

13
src/main/java/com/olympus/apollo/dto/GitPullOutput.java Normal file
View File

@@ -0,0 +1,13 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.Map;
@Setter @Getter
public class GitPullOutput {
private Map<String,String > changes;
private String repoName;
private String message;
}

2
src/main/java/com/olympus/apollo/models/KSGitInfo.java
View File

@@ -1,6 +1,7 @@
package com.olympus.apollo.models;
import java.util.Date;
import java.util.HashMap;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
@@ -20,6 +21,7 @@ public class KSGitInfo {
private String commitId;
private String repoPath;
private KSGitIngestionInfo ksGitIngestionInfo;
private HashMap<String, String> gitModifiedFiles;
private String ingestionStatus;
private Date ingestionDate;
private String ingestionDateFormat;

6
src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java
View File

@@ -1,8 +1,11 @@
package com.olympus.apollo.repository;
import java.util.List;
import java.util.Optional;
import com.olympus.apollo.models.VectorStore;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.rest.core.annotation.RepositoryRestResource;
import org.springframework.stereotype.Repository;
import org.springframework.web.bind.annotation.CrossOrigin;
@@ -12,4 +15,7 @@ import com.olympus.apollo.models.KSGitInfo;
@Repository
public interface KSGitInfoRepository extends MongoRepository<KSGitInfo, String> {
Optional<KSGitInfo> findByRepoName(String repoName);
@Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsApplicationName': ?1}")
Optional<KSGitInfo> findByMetadataAndRepoName(String repoName, String KsApplicationName);
}

5
src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java
View File

@@ -8,6 +8,7 @@ import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
@Repository
@@ -24,4 +25,8 @@ public interface VectorStoreRepository extends MongoRepository<VectorStore, Stri
@Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}")
List<VectorStore> findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
@Query("{'metadata.filePath': ?0}")
Optional<VectorStore> findByFilePath(String filePath);
}

733
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -1,6 +1,7 @@
package com.olympus.apollo.services;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
@@ -12,7 +13,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.olympus.apollo.repository.VectorStoreRepository;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.treewalk.TreeWalk;
@@ -34,18 +37,24 @@ import com.olympus.apollo.repository.KSGitInfoRepository;
@Service
public class GitRepositoryIngestor {
private final VectorStore vectorStore;
private final VectorStore vectorStore;
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@Value("${gitlab.path}")
private String localRepoPath;
public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Async
public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
@Async
public CompletableFuture<Void> ingestGitRepository(String repo) {
//String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
//String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
@@ -66,338 +75,474 @@ public class GitRepositoryIngestor {
}
private void ingestRepo(String repo, KSGitInfo ksGitInfo) {
String repoPath = "/mnt/apollo_storage/repository" +"\\"+ repo + "\\";
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
private void ingestRepo(String repo, KSGitInfo ksGitInfo) {
String repoPath = localRepoPath+"/"+ repo + "/";
//String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\";
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata : " + ingestionInfo.getMetadata());
ksGitInfoRepository.save(ksGitInfo);
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata : " + ingestionInfo.getMetadata());
ksGitInfoRepository.save(ksGitInfo);
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
List<Document> documents = new ArrayList<>();
List<Document> documents = new ArrayList<>();
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
}
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
}
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(), false);
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(), false);
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents: " + splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded");
}
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents: " + splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded");
}
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
} catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e);
}
}
public IngestionOutput checkIngestionStatus(String repoName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repoName);
IngestionOutput ingestionOutput = new IngestionOutput();
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
ingestionOutput.setStatus(ksGitInfo.getIngestionStatus());
ingestionOutput.setMessage("Status Retrieved");
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
ingestionOutput.getIngestedDocumentId().add(ksGitInfo.getId());
}
} else {
ingestionOutput.setStatus("ERROR");
ingestionOutput.setMessage("Document Not Found");
}
return ingestionOutput;
}
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
} catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e);
}
}
public CompletableFuture<Void> ReIngestGitRepository(String repo) throws GitAPIException, IOException {
//String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
//String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
//logger.info("Repository path : " + repoPath);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
reIngestRepo(repo, ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
}
} else {
logger.info("OOPS: Document Not found");
}
return CompletableFuture.completedFuture(null);
}
private void reIngestRepo(String repo, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
private boolean isRelevantFile(String fileName) {
// Add more relevant file extensions as needed
boolean response = false;
if (fileName.endsWith(".java")) {
response = true;
} else if (fileName.endsWith(".py")) {
response = true;
} else if (fileName.endsWith(".js")) {
response = true;
} else if (fileName.endsWith(".vue")) {
response = true;
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
response = true;
}
return response;
}
HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
private Map<String, String> extractMetadata(String fileName, String fileContent) {
Map<String, String> metadata = new HashMap<>();
if (fileName.endsWith(".java")) {
metadata.putAll(extractJavaMetadata(fileContent));
} else if (fileName.endsWith(".py")) {
metadata.putAll(extractPythonMetadata(fileContent));
} else if (fileName.endsWith(".js")) {
metadata.putAll(extractJavaScriptMetadata(fileContent));
} else if (fileName.endsWith(".vue")) {
metadata.putAll(extractVueMetadata(fileContent));
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
metadata.putAll(extractGroovyMetadata(fileContent));
}
return metadata;
}
List<String> filePathsToDelete = new ArrayList<>();
List<String> filePathsToEmbed = new ArrayList<>();
/*
* private Map<String, String> extractJavaMetadata(String fileContent) {
* Map<String, String> metadata = new HashMap<>(); // Simple regex to find class
* names (this is a basic implementation and might // miss some cases) Pattern
* classPattern = Pattern.compile("class\\s+(\\w+)"); Matcher classMatcher =
* classPattern.matcher(fileContent); List<String> classNames = new
* ArrayList<>(); while (classMatcher.find()) {
* classNames.add(classMatcher.group(1)); } metadata.put("classNames",
* String.join(",", classNames)); return metadata; }
*/
for (Map.Entry<String, String> entry : modifiedFiles.entrySet()) {
switch (entry.getKey()) {
case "MODIFY":
String[] modifiedFileList = entry.getValue().split(",");
for (String modifiedFile : modifiedFileList) {
filePathsToDelete.add(modifiedFile);
filePathsToEmbed.add(modifiedFile);
}
break;
case "ADD":
String[] addedFileList = entry.getValue().split(",");
for (String addFile : addedFileList) {
filePathsToEmbed.add(addFile);
}
break;
case "DELETE":
String[] deletedFileList = entry.getValue().split(",");
for (String deletedFile : deletedFileList) {
filePathsToDelete.add(deletedFile);
}
break;
default:
break;
}
for (String fileToDelete : filePathsToDelete) {
Optional<com.olympus.apollo.models.VectorStore> optionalDocument = vectorStoreRepository.findByFilePath(fileToDelete);
if (optionalDocument.isPresent()) {
String vectorStoreId = optionalDocument.get().getId();
vectorStoreRepository.deleteById(vectorStoreId);
}
}
private Map<String, String> extractJavaMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
}
String repoPath = localRepoPath +"/"+ repo + "/";
//String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\"; //need to modify before deploy
logger.info("Repository path : " + repoPath);
// Extract package name
Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);");
Matcher packageMatcher = packagePattern.matcher(fileContent);
if (packageMatcher.find()) {
metadata.put("packageName", packageMatcher.group(1));
}
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata : " + ingestionInfo.getMetadata());
ksGitInfoRepository.save(ksGitInfo);
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
// Extract method names
Pattern methodPattern = Pattern.compile("\\b(?:public|protected|private|static|\\s)\\s*\\w+\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
List<Document> documents = new ArrayList<>();
for (String filePath : filePathsToEmbed) {
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.\\*]+);");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
String[] parts = filePath.split("/");
String fileName = parts[parts.length - 1];
if (isRelevantFile(fileName)) {
boolean fileFound = false;
while (treeWalk.next()) {
if (treeWalk.getPathString().equals(filePath)) {
fileFound = true;
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
return metadata;
}
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
private Map<String, String> extractPythonMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
break;
}
}
if (!fileFound) {
logger.warn("File not found in repository: " + filePath);
}
// Reset TreeWalk to start from the beginning for the next file
treeWalk.reset();
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
}
}
// Extract function names
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+([\\w\\.]+)|\\bfrom\\s+([\\w\\.]+)\\s+import");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
if (importMatcher.group(1) != null) {
importStatements.add(importMatcher.group(1));
} else if (importMatcher.group(2) != null) {
importStatements.add(importMatcher.group(2));
}
}
metadata.put("importStatements", String.join(",", importStatements));
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(), false);
return metadata;
}
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents: " + splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded");
}
private Map<String, String> extractJavaScriptMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract function names
Pattern functionPattern = Pattern.compile("\\bfunction\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
} catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e);
}
}
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+[^;]+\\s+from\\s+['\"]([\\w\\.\\/-]+)['\"]");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
public IngestionOutput checkIngestionStatus(String repoName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repoName);
IngestionOutput ingestionOutput = new IngestionOutput();
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
ingestionOutput.setStatus(ksGitInfo.getIngestionStatus());
ingestionOutput.setMessage("Status Retrieved");
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
ingestionOutput.getIngestedDocumentId().add(ksGitInfo.getId());
}
} else {
ingestionOutput.setStatus("ERROR");
ingestionOutput.setMessage("Repository Not Found");
}
return ingestionOutput;
}
return metadata;
}
private Map<String, String> extractVueMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
private boolean isRelevantFile(String fileName) {
// Add more relevant file extensions as needed
boolean response = false;
if (fileName.endsWith(".java")) {
response = true;
} else if (fileName.endsWith(".py")) {
response = true;
} else if (fileName.endsWith(".js")) {
response = true;
} else if (fileName.endsWith(".vue")) {
response = true;
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
response = true;
}
return response;
}
// Extract component name
Pattern namePattern = Pattern.compile("<script>\\s*export\\s+default\\s+\\{\\s*name:\\s*['\"](\\w+)['\"]");
Matcher nameMatcher = namePattern.matcher(fileContent);
if (nameMatcher.find()) {
metadata.put("componentName", nameMatcher.group(1));
}
private Map<String, String> extractMetadata(String fileName, String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract ref() usage
Pattern refPattern = Pattern.compile("\\bref\\(([^)]*)\\)");
Matcher refMatcher = refPattern.matcher(fileContent);
List<String> refs = new ArrayList<>();
while (refMatcher.find()) {
refs.add(refMatcher.group(1).replaceAll("[\"']", ""));
}
metadata.put("refs", String.join(",", refs));
if (fileName.endsWith(".java")) {
metadata.putAll(extractJavaMetadata(fileContent));
} else if (fileName.endsWith(".py")) {
metadata.putAll(extractPythonMetadata(fileContent));
} else if (fileName.endsWith(".js")) {
metadata.putAll(extractJavaScriptMetadata(fileContent));
} else if (fileName.endsWith(".vue")) {
metadata.putAll(extractVueMetadata(fileContent));
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
metadata.putAll(extractGroovyMetadata(fileContent));
}
return metadata;
}
// Check if template, script, and style tags are present
if (fileContent.contains("<template>")) {
metadata.put("hasTemplate", "true");
}
if (fileContent.contains("<script>")) {
metadata.put("hasScript", "true");
}
if (fileContent.contains("<style>")) {
metadata.put("hasStyle", "true");
}
/*
* private Map<String, String> extractJavaMetadata(String fileContent) {
* Map<String, String> metadata = new HashMap<>(); // Simple regex to find class
* names (this is a basic implementation and might // miss some cases) Pattern
* classPattern = Pattern.compile("class\\s+(\\w+)"); Matcher classMatcher =
* classPattern.matcher(fileContent); List<String> classNames = new
* ArrayList<>(); while (classMatcher.find()) {
* classNames.add(classMatcher.group(1)); } metadata.put("classNames",
* String.join(",", classNames)); return metadata; }
*/
return metadata;
}
private Map<String, String> extractJavaMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
private Map<String, String> extractGroovyMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract package name
Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);");
Matcher packageMatcher = packagePattern.matcher(fileContent);
if (packageMatcher.find()) {
metadata.put("packageName", packageMatcher.group(1));
}
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract method names
Pattern methodPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
// Extract method names
Pattern methodPattern = Pattern.compile("\\b(?:public|protected|private|static|\\s)\\s*\\w+\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.]+)");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.\\*]+);");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
// Extract script variables
Pattern variablePattern = Pattern.compile("\\bdef\\s+(\\w+)");
Matcher variableMatcher = variablePattern.matcher(fileContent);
List<String> scriptVariables = new ArrayList<>();
while (variableMatcher.find()) {
scriptVariables.add(variableMatcher.group(1));
}
metadata.put("scriptVariables", String.join(",", scriptVariables));
return metadata;
}
// Extract stages (for Jenkinsfiles)
Pattern stagePattern = Pattern.compile("\\bstage\\s*\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher stageMatcher = stagePattern.matcher(fileContent);
List<String> stageNames = new ArrayList<>();
while (stageMatcher.find()) {
stageNames.add(stageMatcher.group(1));
}
metadata.put("stageNames", String.join(",", stageNames));
private Map<String, String> extractPythonMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract function names (for Jenkinsfiles)
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract libraries/plugins (for Jenkinsfiles)
Pattern libraryPattern = Pattern.compile("library\\s+\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher libraryMatcher = libraryPattern.matcher(fileContent);
List<String> libraries = new ArrayList<>();
while (libraryMatcher.find()) {
libraries.add(libraryMatcher.group(1));
}
metadata.put("libraries", String.join(",", libraries));
// Extract function names
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
return metadata;
}
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+([\\w\\.]+)|\\bfrom\\s+([\\w\\.]+)\\s+import");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
if (importMatcher.group(1) != null) {
importStatements.add(importMatcher.group(1));
} else if (importMatcher.group(2) != null) {
importStatements.add(importMatcher.group(2));
}
}
metadata.put("importStatements", String.join(",", importStatements));
return metadata;
}
private Map<String, String> extractJavaScriptMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract function names
Pattern functionPattern = Pattern.compile("\\bfunction\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+[^;]+\\s+from\\s+['\"]([\\w\\.\\/-]+)['\"]");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
return metadata;
}
private Map<String, String> extractVueMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract component name
Pattern namePattern = Pattern.compile("<script>\\s*export\\s+default\\s+\\{\\s*name:\\s*['\"](\\w+)['\"]");
Matcher nameMatcher = namePattern.matcher(fileContent);
if (nameMatcher.find()) {
metadata.put("componentName", nameMatcher.group(1));
}
// Extract ref() usage
Pattern refPattern = Pattern.compile("\\bref\\(([^)]*)\\)");
Matcher refMatcher = refPattern.matcher(fileContent);
List<String> refs = new ArrayList<>();
while (refMatcher.find()) {
refs.add(refMatcher.group(1).replaceAll("[\"']", ""));
}
metadata.put("refs", String.join(",", refs));
// Check if template, script, and style tags are present
if (fileContent.contains("<template>")) {
metadata.put("hasTemplate", "true");
}
if (fileContent.contains("<script>")) {
metadata.put("hasScript", "true");
}
if (fileContent.contains("<style>")) {
metadata.put("hasStyle", "true");
}
return metadata;
}
private Map<String, String> extractGroovyMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract method names
Pattern methodPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.]+)");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
// Extract script variables
Pattern variablePattern = Pattern.compile("\\bdef\\s+(\\w+)");
Matcher variableMatcher = variablePattern.matcher(fileContent);
List<String> scriptVariables = new ArrayList<>();
while (variableMatcher.find()) {
scriptVariables.add(variableMatcher.group(1));
}
metadata.put("scriptVariables", String.join(",", scriptVariables));
// Extract stages (for Jenkinsfiles)
Pattern stagePattern = Pattern.compile("\\bstage\\s*\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher stageMatcher = stagePattern.matcher(fileContent);
List<String> stageNames = new ArrayList<>();
while (stageMatcher.find()) {
stageNames.add(stageMatcher.group(1));
}
metadata.put("stageNames", String.join(",", stageNames));
// Extract function names (for Jenkinsfiles)
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract libraries/plugins (for Jenkinsfiles)
Pattern libraryPattern = Pattern.compile("library\\s+\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher libraryMatcher = libraryPattern.matcher(fileContent);
List<String> libraries = new ArrayList<>();
while (libraryMatcher.find()) {
libraries.add(libraryMatcher.group(1));
}
metadata.put("libraries", String.join(",", libraries));
return metadata;
}
}

204
src/main/java/com/olympus/apollo/services/GitService.java Normal file
View File

@@ -0,0 +1,204 @@
package com.olympus.apollo.services;
import com.olympus.apollo.dto.GitCloneOutput;
import com.olympus.apollo.dto.GitPullOutput;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.VectorStoreRepository;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.RefNotFoundException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Service
public class GitService {
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@Value("${gitlab.path}")
private String localPath;
@Value("${gitlab.cloud.token}")
private String cloudGitlabToken;
@Value("${gitlab.onpremises.token}")
private String onPremisesGitlabToken;
private String gitHubToken="23";
public GitCloneOutput cloneRepository(String Source,String repoName,String group, String tokenType) throws GitAPIException {
GitCloneOutput gitCloneOutput= new GitCloneOutput();
String gitlabToken;
String remoteRepoUrl;
switch (tokenType){
case "cloud":
gitlabToken=cloudGitlabToken;
remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
break;
case "onpremises":
gitlabToken=onPremisesGitlabToken;
remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
break;
case "github":
gitlabToken=gitHubToken;
remoteRepoUrl= Source+"/"+group+"/"+repoName+".git";
//String remoteRepoUrl= "https://github.com/automationtester23/shellExecutionThroughAPI.git";
break;
default:
gitCloneOutput.setMessage("tokenType is invalid");
gitCloneOutput.setRepoName(repoName);
return gitCloneOutput;
}
System.out.println("remoteUrl : "+remoteRepoUrl);
System.out.println("gitlab token : "+gitlabToken);
File cloneDirectory = new File(localPath,repoName);
System.out.println("cloneDirectory : "+cloneDirectory);
if(tokenType.equals("github")){
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
}else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",gitlabToken)).call();
}
checkOutRepository(repoName,"master");
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage(repoName +" cloned successfully");
return gitCloneOutput;
}
public String checkOutRepository(String repoName, String branchName) {
File repoDirectory = new File(localPath,repoName);
try(Git git = Git.open(repoDirectory)) {
git.checkout().setName(branchName).setCreateBranch(false).call();
return branchName+" checkout successful";
} catch (RefNotFoundException e) {
try(Git git = Git.open(repoDirectory)) {
git.checkout().setName("origin/"+branchName).setCreateBranch(false).call();
return branchName+" checkout successful";
} catch (RefNotFoundException ex) {
return "checkout failed with error "+e.getMessage();
}catch (Exception ex2){
System.out.println("An error occurred: "+ ex2.getMessage());
return "checkout failed with error "+ ex2.getMessage();
}
} catch (Exception e){
System.out.println("An error occurred: "+ e.getMessage());
return "checkout failed with error "+e.getMessage();
}
}
public GitPullOutput pullChanges(String repoName) {
File repoDirectory = new File(localPath,repoName);
GitPullOutput gitPullOutput =new GitPullOutput();
Map<String,String> gitdiff=null;
try(Git git = Git.open(repoDirectory)) {
String message = checkOutRepository(repoName,"master");
System.out.println("message from checkoutRepository : "+message);
gitdiff = getModifiedFiles(git);
HashMap<String,String> modifiedFiles = new HashMap<>();
for (Map.Entry<String,String> entry: gitdiff.entrySet()){
System.out.println(entry.getKey()+" "+entry.getValue());
modifiedFiles.put(entry.getKey(),entry.getValue());
}
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByMetadataAndRepoName(repoName,repoName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
ksGitInfo.setGitModifiedFiles(modifiedFiles);
ksGitInfoRepository.save(ksGitInfo);
}else {
System.out.println("Records not present");
}
git.pull().call();
/*
if (commitId != null && !commitId.isEmpty()) {
String message1 = checkOutRepository(repoName,commitId);
System.out.println("message1 : "+message1);
}*/
gitPullOutput.setRepoName(repoName);
gitPullOutput.setMessage("repo pulled");
gitPullOutput.setChanges(gitdiff);
} catch (RefNotFoundException e) {
System.out.println("checkout failed with error "+e.getMessage());
gitPullOutput.setRepoName(repoName);
gitPullOutput.setMessage("error occurred "+e.getMessage());
gitPullOutput.setChanges(gitdiff);
} catch (Exception e){
System.out.println("An error occurred: "+ e.getMessage());
gitPullOutput.setRepoName(repoName);
gitPullOutput.setMessage("error occurred "+e.getMessage());
gitPullOutput.setChanges(gitdiff);
}
return gitPullOutput;
}
private Map<String,String> getModifiedFiles(Git git) throws GitAPIException, IOException {
Repository repository = git.getRepository();
git.fetch().call();
AbstractTreeIterator oldTreeParser = prepareTreeParser(repository, "HEAD");
AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/master");
Map<String, String> modifiedFiles = new HashMap<>();
try (DiffFormatter formatter = new DiffFormatter(System.out)) {
formatter.setRepository(repository);
List<DiffEntry> entries = formatter.scan(oldTreeParser, newTreeParser);
for (DiffEntry entry : entries) {
String changeType = entry.getChangeType().name();
String filePath = entry.getNewPath();
if(modifiedFiles.containsKey(changeType)){
String existingFilePaths = modifiedFiles.get(changeType);
modifiedFiles.put(changeType,existingFilePaths+","+filePath);
}else {
modifiedFiles.put(changeType,filePath);
}
}
}
return modifiedFiles;
}
private AbstractTreeIterator prepareTreeParser(Repository repository, String ref) throws IOException, IOException {
ObjectId id = repository.resolve(ref);
try (var reader = repository.newObjectReader()) {
return new CanonicalTreeParser(null, reader, repository.resolve(id.name() + "^{tree}"));
}
}
// Method to extract the protocol part
private static String getProtocol(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(0, protocolEndIndex + 3) : "";
}
// Method to extract the domain part
private static String getDomain(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(protocolEndIndex + 3) : url;
}
}

6
src/main/resources/application.properties
View File

@@ -28,4 +28,8 @@ spring.servlet.multipart.max-file-size=5000000MB
spring.servlet.multipart.max-request-size=500000MB
#path to the repository
ingestion.repository.basepath=C:\\Users\\andrea.terzani\\dev\\Olympus
ingestion.repository.basepath=C:\\Users\\andrea.terzani\\dev\\Olympus
gitlab.token=
#gitlab.path=C:\\repos\\olympus_ai\\gitClone
gitlab.path=/mnt/apollo_storage/repository;