admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multiple branch capability added

Browse Source
This commit is contained in:
sumedh
2024-09-05 13:01:56 +05:30
parent cc2613be15
commit 1679ab8dba
4 changed files with 71 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java
View File

@@ -32,4 +32,6 @@ public interface VectorStoreRepository extends MongoRepository<VectorStore, Stri
@Query("{'metadata.filePath': ?0}")
Optional<VectorStore> findByFilePath(String filePath);
@Query("{'metadata.KsApplicationName': ?0, 'metadata.KsBranch': ?1, 'metadata.filePath': ?2}")
Optional<VectorStore> findByKsapplicationNameKsBranchFilePath(String ksApplicationName,String ksBranch,String filePath);
}

21
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -48,6 +48,9 @@ public class GitRepositoryIngestor {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Autowired
private GitService gitService;
public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
@@ -71,9 +74,10 @@ public class GitRepositoryIngestor {
}
private void ingestRepo(String repo,String branchName, KSGitInfo ksGitInfo) {
String repoPath = basePath+"/"+branchName+"/"+ repo + "/";
//String repoPath = basePath + "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy
private void ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) {
String repoPath = basePath+"/"+ repoName + "/";
//String repoPath = basePath + "\\" + repoName + "\\"; //need to modify before deploy
gitService.checkOutRepository(repoName,branchName);
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
@@ -132,7 +136,7 @@ public class GitRepositoryIngestor {
}
public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
@@ -146,7 +150,7 @@ public class GitRepositoryIngestor {
return CompletableFuture.completedFuture(null);
}
private void reIngestRepo(String repo,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
@@ -180,15 +184,16 @@ public class GitRepositoryIngestor {
break;
}
for (String fileToDelete : filePathsToDelete) {
Optional<com.olympus.apollo.models.VectorStore> optionalDocument = vectorStoreRepository.findByFilePath(fileToDelete);
Optional<com.olympus.apollo.models.VectorStore> optionalDocument = vectorStoreRepository.findByKsapplicationNameKsBranchFilePath(repoName,branchName,fileToDelete);
if (optionalDocument.isPresent()) {
String vectorStoreId = optionalDocument.get().getId();
vectorStoreRepository.deleteById(vectorStoreId);
}
}
}
String repoPath = basePath+"/"+branchName +"/"+ repo + "/";
//String repoPath = basePath+ "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy
gitService.checkOutRepository(repoName,branchName);
String repoPath = basePath+"/"+ repoName + "/";
//String repoPath = basePath+ "\\" + repoName + "\\"; //need to modify before deploy
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {

66
src/main/java/com/olympus/apollo/services/GitService.java
View File

@@ -11,8 +11,10 @@ import org.eclipse.jgit.api.errors.RefNotFoundException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.springframework.beans.factory.annotation.Autowired;
@@ -69,18 +71,24 @@ public class GitService {
return gitCloneOutput;
}
System.out.println("remoteUrl : " + remoteRepoUrl);
System.out.println("gitlab token : " + gitlabToken);
String localPath = basePath + "/" + branchName;
//String localPath = basePath + "\\" + branchName; //need to change before deploy
Files.createDirectories(Paths.get(localPath));
File cloneDirectory = new File(localPath, repoName);
System.out.println("cloneDirectory : " + cloneDirectory);
if (tokenType.equals("github")) {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
} else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call();
//System.out.println("gitlab token : " + gitlabToken);
String localPath = basePath;
System.out.println("localpath"+basePath);
File cloneDirectory = new File(localPath,repoName);
//Files.createDirectories(Paths.get(localPath));
//File cloneDirectory = new File(localPath, repoName);
//System.out.println("cloneDirectory : " + cloneDirectory);
if(!cloneDirectory.exists()){
if (tokenType.equals("github")) {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
} else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call();
}
}else {
System.out.println("Directory already exists. Skipping clone.");
}
checkOutRepository(repoName, branchName);
gitCloneOutput.setRepoName(repoName);
@@ -94,11 +102,18 @@ public class GitService {
}
public String checkOutRepository(String repoName, String branchName) {
String localPath = basePath+"/"+branchName;
//String localPath = basePath+"\\"+branchName; //need to change before deploy
String localPath = basePath;
File repoDirectory = new File(localPath,repoName);
try(Git git = Git.open(repoDirectory)) {
git.checkout().setName(branchName).setCreateBranch(false).call();
Repository repository = git.getRepository();
if(!branchExists(branchName,repository)) {
FetchResult fetchResult = git.fetch().call();
System.out.println("Fetch result: " + fetchResult.getMessages());
git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/"+branchName).call();
}else {
git.checkout().setName(branchName).setCreateBranch(false).call();
System.out.println("Checked out existing branch: " + branchName);
}
return branchName+" checkout successful";
} catch (RefNotFoundException e) {
try(Git git = Git.open(repoDirectory)) {
@@ -116,9 +131,28 @@ public class GitService {
}
}
private static boolean branchExists(String branchName,Repository repository){
try{
List<Ref> refs = repository.getRefDatabase().getRefs();
for(Ref ref : refs){
System.out.println("refs "+ref.getName());
}
for(Ref ref : refs){
if(ref.getName().equals("refs/heads/"+branchName)){
return true;
}
}
return false;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public GitPullOutput pullChanges(String repoName,String branchName) {
String localPath= basePath+"/"+branchName;
//String localPath= basePath+"\\"+branchName; //need ot change before deploy
String localPath= basePath;
File repoDirectory = new File(localPath,repoName);
GitPullOutput gitPullOutput =new GitPullOutput();
Map<String,String> gitdiff=null;
@@ -135,7 +169,7 @@ public class GitService {
modifiedFiles.put(entry.getKey(),entry.getValue());
}
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByMetadataAndRepoName(repoName,repoName);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repoName,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
ksGitInfo.setGitModifiedFiles(modifiedFiles);

12
src/main/resources/application.properties
View File

@@ -9,19 +9,19 @@ server.port=8082
#spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect
spring.ai.vectorstore.mongodb.uri=mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus
spring.ai.vectorstore.mongodb.uri=
spring.data.mongodb.uri=mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus
spring.data.mongodb.database=olympus
spring.data.mongodb.username=olympus_adm
spring.data.mongodb.password=26111979
spring.data.mongodb.uri=
spring.data.mongodb.database=
spring.data.mongodb.username=
spring.data.mongodb.password=
spring.ai.vectorstore.mongodb.indexName=vector_index
spring.ai.vectorstore.mongodb.collection-name=vector_store
spring.ai.vectorstore.mongodb.initialize-schema=false
# API key if needed, e.g. OpenAI
spring.ai.openai.api-key=sk-proj-j3TFJ0h348DIzMrYYfyUT3BlbkFJjk4HMc8A2ux2Asg8Y7H1
spring.ai.openai.api-key=
#size filter
spring.servlet.multipart.max-file-size=5000000MB