diff --git a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java index 391f60b..fd3f009 100644 --- a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java +++ b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java @@ -32,4 +32,6 @@ public interface VectorStoreRepository extends MongoRepository findByFilePath(String filePath); + @Query("{'metadata.KsApplicationName': ?0, 'metadata.KsBranch': ?1, 'metadata.filePath': ?2}") + Optional findByKsapplicationNameKsBranchFilePath(String ksApplicationName,String ksBranch,String filePath); } diff --git a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java index d515cf6..40b05e6 100644 --- a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +++ b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java @@ -48,6 +48,9 @@ public class GitRepositoryIngestor { @Autowired private VectorStoreRepository vectorStoreRepository; + @Autowired + private GitService gitService; + public GitRepositoryIngestor(VectorStore vectorStore) { this.vectorStore = vectorStore; } @@ -71,9 +74,10 @@ public class GitRepositoryIngestor { } - private void ingestRepo(String repo,String branchName, KSGitInfo ksGitInfo) { - String repoPath = basePath+"/"+branchName+"/"+ repo + "/"; - //String repoPath = basePath + "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy + private void ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) { + String repoPath = basePath+"/"+ repoName + "/"; + //String repoPath = basePath + "\\" + repoName + "\\"; //need to modify before deploy + gitService.checkOutRepository(repoName,branchName); logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { ksGitInfo.setIngestionStatus("IN PROGRESS"); @@ -132,7 +136,7 @@ public class GitRepositoryIngestor { } public CompletableFuture ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException { - Optional optionalDocument = ksGitInfoRepository.findByRepoName(repo); + Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) { @@ -146,7 +150,7 @@ public class GitRepositoryIngestor { return CompletableFuture.completedFuture(null); } - private void reIngestRepo(String repo,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException { + private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException { HashMap modifiedFiles = ksGitInfo.getGitModifiedFiles(); @@ -180,15 +184,16 @@ public class GitRepositoryIngestor { break; } for (String fileToDelete : filePathsToDelete) { - Optional optionalDocument = vectorStoreRepository.findByFilePath(fileToDelete); + Optional optionalDocument = vectorStoreRepository.findByKsapplicationNameKsBranchFilePath(repoName,branchName,fileToDelete); if (optionalDocument.isPresent()) { String vectorStoreId = optionalDocument.get().getId(); vectorStoreRepository.deleteById(vectorStoreId); } } } - String repoPath = basePath+"/"+branchName +"/"+ repo + "/"; - //String repoPath = basePath+ "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy + gitService.checkOutRepository(repoName,branchName); + String repoPath = basePath+"/"+ repoName + "/"; + //String repoPath = basePath+ "\\" + repoName + "\\"; //need to modify before deploy logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { diff --git a/src/main/java/com/olympus/apollo/services/GitService.java b/src/main/java/com/olympus/apollo/services/GitService.java index b935aff..eaf997f 100644 --- a/src/main/java/com/olympus/apollo/services/GitService.java +++ b/src/main/java/com/olympus/apollo/services/GitService.java @@ -11,8 +11,10 @@ import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.springframework.beans.factory.annotation.Autowired; @@ -69,18 +71,24 @@ public class GitService { return gitCloneOutput; } System.out.println("remoteUrl : " + remoteRepoUrl); - System.out.println("gitlab token : " + gitlabToken); - String localPath = basePath + "/" + branchName; - //String localPath = basePath + "\\" + branchName; //need to change before deploy - Files.createDirectories(Paths.get(localPath)); - File cloneDirectory = new File(localPath, repoName); - System.out.println("cloneDirectory : " + cloneDirectory); - if (tokenType.equals("github")) { - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call(); - } else { - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call(); + //System.out.println("gitlab token : " + gitlabToken); + String localPath = basePath; + System.out.println("localpath"+basePath); + File cloneDirectory = new File(localPath,repoName); + //Files.createDirectories(Paths.get(localPath)); + //File cloneDirectory = new File(localPath, repoName); + //System.out.println("cloneDirectory : " + cloneDirectory); + if(!cloneDirectory.exists()){ + if (tokenType.equals("github")) { + Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call(); + } else { + Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call(); + } + }else { + System.out.println("Directory already exists. Skipping clone."); } + checkOutRepository(repoName, branchName); gitCloneOutput.setRepoName(repoName); @@ -94,11 +102,18 @@ public class GitService { } public String checkOutRepository(String repoName, String branchName) { - String localPath = basePath+"/"+branchName; - //String localPath = basePath+"\\"+branchName; //need to change before deploy + String localPath = basePath; File repoDirectory = new File(localPath,repoName); try(Git git = Git.open(repoDirectory)) { - git.checkout().setName(branchName).setCreateBranch(false).call(); + Repository repository = git.getRepository(); + if(!branchExists(branchName,repository)) { + FetchResult fetchResult = git.fetch().call(); + System.out.println("Fetch result: " + fetchResult.getMessages()); + git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/"+branchName).call(); + }else { + git.checkout().setName(branchName).setCreateBranch(false).call(); + System.out.println("Checked out existing branch: " + branchName); + } return branchName+" checkout successful"; } catch (RefNotFoundException e) { try(Git git = Git.open(repoDirectory)) { @@ -116,9 +131,28 @@ public class GitService { } } + private static boolean branchExists(String branchName,Repository repository){ + try{ + List refs = repository.getRefDatabase().getRefs(); + + for(Ref ref : refs){ + System.out.println("refs "+ref.getName()); + } + + for(Ref ref : refs){ + if(ref.getName().equals("refs/heads/"+branchName)){ + return true; + } + } + return false; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public GitPullOutput pullChanges(String repoName,String branchName) { - String localPath= basePath+"/"+branchName; - //String localPath= basePath+"\\"+branchName; //need ot change before deploy + String localPath= basePath; + File repoDirectory = new File(localPath,repoName); GitPullOutput gitPullOutput =new GitPullOutput(); Map gitdiff=null; @@ -135,7 +169,7 @@ public class GitService { modifiedFiles.put(entry.getKey(),entry.getValue()); } - Optional optionalDocument = ksGitInfoRepository.findByMetadataAndRepoName(repoName,repoName); + Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repoName,branchName); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); ksGitInfo.setGitModifiedFiles(modifiedFiles); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ecd45cc..c426c9a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -9,19 +9,19 @@ server.port=8082 #spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect -spring.ai.vectorstore.mongodb.uri=mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus +spring.ai.vectorstore.mongodb.uri= -spring.data.mongodb.uri=mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus -spring.data.mongodb.database=olympus -spring.data.mongodb.username=olympus_adm -spring.data.mongodb.password=26111979 +spring.data.mongodb.uri= +spring.data.mongodb.database= +spring.data.mongodb.username= +spring.data.mongodb.password= spring.ai.vectorstore.mongodb.indexName=vector_index spring.ai.vectorstore.mongodb.collection-name=vector_store spring.ai.vectorstore.mongodb.initialize-schema=false # API key if needed, e.g. OpenAI -spring.ai.openai.api-key=sk-proj-j3TFJ0h348DIzMrYYfyUT3BlbkFJjk4HMc8A2ux2Asg8Y7H1 +spring.ai.openai.api-key= #size filter spring.servlet.multipart.max-file-size=5000000MB