admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

same repository multiple branches clone functionality implemented

Browse Source
This commit is contained in:
sumedh
2024-08-29 22:57:24 +05:30
parent 35bd7e3cf5
commit 47af964bd2
16 changed files with 153 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@@ -31,3 +31,10 @@ build/
### VS Code ###
.vscode/
##changes in filepath before deploy
src/main/java/com/olympus/apollo/services/StorageProperties.java
src/main/resources/application.properties
src/main/java/com/olympus/apollo/services/GitService.java
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java

17
pom.xml
View File

@@ -28,6 +28,7 @@
</scm>
<properties>
<java.version>21</java.version>
<spring-ai.version>1.0.0-M1</spring-ai.version>
</properties>
<dependencies>
@@ -53,13 +54,6 @@
<type>pom</type>
</dependency>
<!--<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-redis-store-spring-boot-starter</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-rest</artifactId>
@@ -141,15 +135,6 @@
<updatePolicy>never</updatePolicy>
</snapshots>
</repository>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
<updatePolicy>never</updatePolicy>
</releases>
</repository>
</repositories>
<build>

24
src/main/java/com/olympus/apollo/config/AsyncConfiguration.java Normal file
View File

@@ -0,0 +1,24 @@
package com.olympus.apollo.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
@Configuration
@EnableAsync //using this annotation spring boot will internally search which all methods have annotation async
public class AsyncConfiguration {
@Bean("asyncTaskExecutor")
public Executor asyncTaskExecutor(){
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setCorePoolSize(4); //these threads are kept alive even if they are idle
taskExecutor.setQueueCapacity(150); //maximum number of tasks that can be held in queue if number gets exceeded then it will create new thread as per maximum pool size defined
taskExecutor.setMaxPoolSize(4);
taskExecutor.setThreadNamePrefix("AsyncTaskThread-");
taskExecutor.initialize();
return taskExecutor;
}
}

15
src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java
View File

@@ -10,6 +10,7 @@ import com.olympus.apollo.dto.*;
import com.olympus.apollo.services.GitService;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
@@ -19,7 +20,6 @@ import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.KSGitIngestionInfoRepository;
import com.olympus.apollo.services.KSGitInfoService;
@CrossOrigin(origins = "http://localhost:5173")
@RestController
@RequestMapping("/fe-api/ks_git_repos")
public class KSGitController {
@@ -32,6 +32,9 @@ public class KSGitController {
@Autowired
private GitService gitService;
@Value("${gitlab.path}")
private String basePath;
@GetMapping("")
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
@@ -81,7 +84,7 @@ public class KSGitController {
ksGitInfo.setRepoName(gitCloneInput.getRepoName());
ksGitInfo.setBranch(gitCloneInput.getBranch());
ksGitInfo.setCommitId(gitCloneInput.getCommitId());
ksGitInfo.setRepoPath(gitCloneInput.getRepoPath());
ksGitInfo.setRepoPath(basePath+"/"+gitCloneInput.getBranch());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
@@ -107,7 +110,7 @@ public class KSGitController {
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getBranch(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
}
/*
curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \
@@ -118,9 +121,9 @@ public class KSGitController {
*/
//pull latest changes from master branch
@GetMapping("/pullchanges/{repoName}")
public GitPullOutput gitPull(@PathVariable String repoName){
return gitService.pullChanges(repoName);
@GetMapping("/pullchanges")
public GitPullOutput gitPull(@RequestParam String repoName,@RequestParam String branchName){
return gitService.pullChanges(repoName,branchName);
}
}

1
src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java
View File

@@ -11,7 +11,6 @@ import com.olympus.apollo.repository.KSDocumentRepository;
@RestController
@RequestMapping("/fe-api/ksdocuments")
@CrossOrigin(origins = "http://localhost:5173")
public class KsDocumentController {
@Autowired

10
src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java
View File

@@ -7,13 +7,13 @@ import com.olympus.apollo.models.VectorStore;
import com.olympus.apollo.repository.VectorStoreRepository;
import com.olympus.apollo.services.DeletionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
@RequestMapping("/fe-api/vector-store")
@CrossOrigin(origins = "http://localhost:5173")
public class VectorStoreController {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@@ -45,15 +45,15 @@ public class VectorStoreController {
}
@PostMapping("/deleteRecords")
public String deleteRecords(@RequestBody DeletionRequest deletionRequest){
public ResponseEntity<String> deleteRecords(@RequestBody DeletionRequest deletionRequest){
deletionService.deleteRecords(deletionRequest);
return "Records Deleted Successfully";
return ResponseEntity.ok("Request In Working");
}
@PostMapping("/deleteGitRecords")
public String deleteGitRecords(@RequestBody DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest){
public ResponseEntity<String> deleteGitRecords(@RequestBody DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest){
deletionService.deleteRecordsOfGitRepo(deleteGitRepoDetailsRequest);
return "Git Records Deleted Successfully";
return ResponseEntity.ok("Git Records Deletion IN PROGRESS");
}
}

2
src/main/java/com/olympus/apollo/controllers/KSFileController.java
View File

@@ -16,7 +16,7 @@ import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.services.StorageFileNotFoundException;
import com.olympus.apollo.services.StorageService;
import com.olympus.apollo.dto.FileUploadDTO;
@CrossOrigin
@RestController
public class KSFileController {
@Autowired

25
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -54,10 +54,10 @@ public class TestController {
return "Deleted";
}
@GetMapping("test/ingest_repo/{repoName}")
public ResponseEntity<String> ingestRepo(@PathVariable String repoName) {
@GetMapping("test/ingest_repo")
public ResponseEntity<String> ingestRepo(@RequestParam String repoName,@RequestParam String branchName) {
try {
gitRepositoryIngestor.ingestGitRepository(repoName);
gitRepositoryIngestor.ingestGitRepository(repoName,branchName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
@@ -68,11 +68,11 @@ public class TestController {
}
}
@GetMapping("test/reingest_repo/{repoName}")
public ResponseEntity<String> ReIngestRepo(@PathVariable String repoName) {
@GetMapping("test/reingest_repo")
public ResponseEntity<String> ReIngestRepo(@RequestParam String repoName,@RequestParam String branchName) {
try {
gitService.pullChanges(repoName);
gitRepositoryIngestor.ReIngestGitRepository(repoName);
gitService.pullChanges(repoName,branchName);
gitRepositoryIngestor.ReIngestGitRepository(repoName,branchName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
@@ -82,15 +82,12 @@ public class TestController {
.body("Error starting ingestion: " + e.getMessage());
}
}
/*
curl --location 'http://localhost:8082/test/reingest_repo/shellExecutionThroughAPI' \
--header 'Authorization: Bearer '
*/
@GetMapping("test/check_ingestion_status/{repoName}")
public ResponseEntity<IngestionOutput> checkIngestionStatus(@PathVariable String repoName) {
@GetMapping("test/check_ingestion_status")
public ResponseEntity<IngestionOutput> checkIngestionStatus(@RequestParam String repoName,@RequestParam String branchName) {
try {
IngestionOutput ingestionOutput = gitRepositoryIngestor.checkIngestionStatus(repoName);
IngestionOutput ingestionOutput = gitRepositoryIngestor.checkIngestionStatus(repoName,branchName);
return ResponseEntity.ok(ingestionOutput);
} catch (Exception e) {

1
src/main/java/com/olympus/apollo/dto/DeleteGitRepoDetailsRequest.java
View File

@@ -11,4 +11,5 @@ public class DeleteGitRepoDetailsRequest {
private String ksDocSource;
private String ksFileSource;
private String ksApplicationName;
private String ksBranch;
}

3
src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java
View File

@@ -18,4 +18,7 @@ public interface KSGitInfoRepository extends MongoRepository<KSGitInfo, String>
@Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsApplicationName': ?1}")
Optional<KSGitInfo> findByMetadataAndRepoName(String repoName, String KsApplicationName);
@Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsBranch': ?1}")
Optional<KSGitInfo> findByRepoNameAndBranchName(String repoName, String KsApplicationName);
}

5
src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java
View File

@@ -23,8 +23,11 @@ public interface VectorStoreRepository extends MongoRepository<VectorStore, Stri
})
List<VectorStoreDetails> findAllDetails();
@Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3, 'metadata.KsBranch': ?4}")
List<VectorStore> findGitVectorByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName, String ksBranch);
@Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}")
List<VectorStore> findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
List<VectorStore> findDocumentVectorByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
@Query("{'metadata.filePath': ?0}")
Optional<VectorStore> findByFilePath(String filePath);

26
src/main/java/com/olympus/apollo/services/DeletionService.java
View File

@@ -12,6 +12,7 @@ import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.util.List;
@@ -37,6 +38,7 @@ public class DeletionService {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Async("asyncTaskExecutor")
public void deleteRecords(DeletionRequest deletionRequest) {
try {
boolean KSDocumentExists = deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty() && ksDocumentRepository.existsById(deletionRequest.getKsDocumentId());
@@ -44,53 +46,53 @@ public class DeletionService {
boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null;
List<VectorStore> vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of();
List<VectorStore> vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findDocumentVectorByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of();
if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) {
if (deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty()) {
ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId());
logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId());
logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()+" "+Thread.currentThread().getName());
}
if (deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty()) {
ksIngestionInfoRepository.deleteById(deletionRequest.getKsIngestionInfoId());
logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId());
logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId()+" "+Thread.currentThread().getName());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", store.getId());
logger.info("VectorStore with id {} deleted successfully.", store.getId()+" "+Thread.currentThread().getName());
}
logger.info("All records deleted successfully.");
} else {
if (!KSDocumentExists) {
logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId());
logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()+" "+Thread.currentThread().getName());
} else if (!KSIngestionInfoExists) {
logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId());
logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId()+" "+Thread.currentThread().getName());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
logger.warn("No VectorStore Data available",Thread.currentThread().getName());
}
}
} catch (Exception e) {
logger.error("An error occurred while deleting records: ", e);
logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName());
throw new RuntimeException("An error occurred while deleting records", e);
}
}
@Async("asyncTaskExecutor")
public void deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) {
try {
boolean KSGitInfoExists = deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty() && ksGitInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KSGitInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitInfoId());
boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null;
boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null && deleteGitRepoDetailsRequest.getKsBranch() != null;
Optional<KSGitInfo> ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId());
String ingestionStatus = ksGitInfo.get().getIngestionStatus();
logger.info("Ingestion Status is {}.",ingestionStatus);
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName()) : List.of();
logger.info("Ingestion Status is {}.",ingestionStatus+" "+Thread.currentThread().getName());
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findGitVectorByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsBranch()) : List.of();
if (KSGitInfoExists && KSGitIngestionInfoExists) {
if(ingestionStatus.equals("ERROR")){

39
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -40,7 +40,7 @@ public class GitRepositoryIngestor {
private final VectorStore vectorStore;
@Value("${gitlab.path}")
private String localRepoPath;
private String basePath;
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@@ -55,16 +55,12 @@ public class GitRepositoryIngestor {
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
@Async
public CompletableFuture<Void> ingestGitRepository(String repo) {
//String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
//String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
//logger.info("Repository path : " + repoPath);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
public CompletableFuture<Void> ingestGitRepository(String repo,String branchName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) {
ingestRepo(repo, ksGitInfo);
ingestRepo(repo,branchName, ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
}
@@ -75,9 +71,9 @@ public class GitRepositoryIngestor {
}
private void ingestRepo(String repo, KSGitInfo ksGitInfo) {
String repoPath = localRepoPath+"/"+ repo + "/";
//String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\";
private void ingestRepo(String repo,String branchName, KSGitInfo ksGitInfo) {
String repoPath = basePath+"/"+branchName+"/"+ repo + "/";
//String repoPath = basePath + "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
@@ -135,18 +131,14 @@ public class GitRepositoryIngestor {
}
}
public CompletableFuture<Void> ReIngestGitRepository(String repo) throws GitAPIException, IOException {
//String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
//String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
//logger.info("Repository path : " + repoPath);
public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
reIngestRepo(repo, ksGitInfo);
reIngestRepo(repo, branchName,ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
logger.info("OOPS: Document is not igested");
}
} else {
logger.info("OOPS: Document Not found");
@@ -154,7 +146,7 @@ public class GitRepositoryIngestor {
return CompletableFuture.completedFuture(null);
}
private void reIngestRepo(String repo, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
private void reIngestRepo(String repo,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
@@ -194,10 +186,9 @@ public class GitRepositoryIngestor {
vectorStoreRepository.deleteById(vectorStoreId);
}
}
}
String repoPath = localRepoPath +"/"+ repo + "/";
//String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\"; //need to modify before deploy
String repoPath = basePath+"/"+branchName +"/"+ repo + "/";
//String repoPath = basePath+ "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
@@ -270,8 +261,8 @@ public class GitRepositoryIngestor {
}
}
public IngestionOutput checkIngestionStatus(String repoName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repoName);
public IngestionOutput checkIngestionStatus(String repoName,String branchName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repoName,branchName);
IngestionOutput ingestionOutput = new IngestionOutput();
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();

100
src/main/java/com/olympus/apollo/services/GitService.java
View File

@@ -21,6 +21,8 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -33,7 +35,7 @@ public class GitService {
private KSGitInfoRepository ksGitInfoRepository;
@Value("${gitlab.path}")
private String localPath;
private String basePath;
@Value("${gitlab.cloud.token}")
private String cloudGitlabToken;
@@ -43,49 +45,57 @@ public class GitService {
private String gitHubToken="23";
public GitCloneOutput cloneRepository(String Source,String repoName,String group, String tokenType) throws GitAPIException {
GitCloneOutput gitCloneOutput= new GitCloneOutput();
String gitlabToken;
String remoteRepoUrl;
switch (tokenType){
case "cloud":
gitlabToken=cloudGitlabToken;
remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
break;
case "onpremises":
gitlabToken=onPremisesGitlabToken;
remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git";
break;
case "github":
gitlabToken=gitHubToken;
remoteRepoUrl= Source+"/"+group+"/"+repoName+".git";
//String remoteRepoUrl= "https://github.com/automationtester23/shellExecutionThroughAPI.git";
break;
default:
gitCloneOutput.setMessage("tokenType is invalid");
gitCloneOutput.setRepoName(repoName);
return gitCloneOutput;
public GitCloneOutput cloneRepository(String Source,String repoName,String branchName,String group, String tokenType) throws GitAPIException {
GitCloneOutput gitCloneOutput = new GitCloneOutput();
try {
String gitlabToken;
String remoteRepoUrl;
switch (tokenType) {
case "cloud":
gitlabToken = cloudGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "onpremises":
gitlabToken = onPremisesGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "github":
gitlabToken = gitHubToken;
remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git";
break;
default:
gitCloneOutput.setMessage("tokenType is invalid");
gitCloneOutput.setRepoName(repoName);
return gitCloneOutput;
}
System.out.println("remoteUrl : " + remoteRepoUrl);
System.out.println("gitlab token : " + gitlabToken);
String localPath = basePath + "/" + branchName;
//String localPath = basePath + "\\" + branchName; //need to change before deploy
Files.createDirectories(Paths.get(localPath));
File cloneDirectory = new File(localPath, repoName);
System.out.println("cloneDirectory : " + cloneDirectory);
if (tokenType.equals("github")) {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
} else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call();
}
checkOutRepository(repoName, branchName);
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage(repoName + "With Branch " + branchName + " cloned successfully");
}catch (Exception e){
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage("Error occured : "+e.getMessage());
}
System.out.println("remoteUrl : "+remoteRepoUrl);
System.out.println("gitlab token : "+gitlabToken);
File cloneDirectory = new File(localPath,repoName);
System.out.println("cloneDirectory : "+cloneDirectory);
if(tokenType.equals("github")){
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
}else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",gitlabToken)).call();
}
checkOutRepository(repoName,"master");
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage(repoName +" cloned successfully");
return gitCloneOutput;
}
public String checkOutRepository(String repoName, String branchName) {
String localPath = basePath+"/"+branchName;
//String localPath = basePath+"\\"+branchName; //need to change before deploy
File repoDirectory = new File(localPath,repoName);
try(Git git = Git.open(repoDirectory)) {
git.checkout().setName(branchName).setCreateBranch(false).call();
@@ -106,15 +116,17 @@ public class GitService {
}
}
public GitPullOutput pullChanges(String repoName) {
public GitPullOutput pullChanges(String repoName,String branchName) {
String localPath= basePath+"/"+branchName;
//String localPath= basePath+"\\"+branchName; //need ot change before deploy
File repoDirectory = new File(localPath,repoName);
GitPullOutput gitPullOutput =new GitPullOutput();
Map<String,String> gitdiff=null;
try(Git git = Git.open(repoDirectory)) {
String message = checkOutRepository(repoName,"master");
String message = checkOutRepository(repoName,branchName);
System.out.println("message from checkoutRepository : "+message);
gitdiff = getModifiedFiles(git);
gitdiff = getModifiedFiles(git,branchName);
HashMap<String,String> modifiedFiles = new HashMap<>();
@@ -155,12 +167,12 @@ public class GitService {
return gitPullOutput;
}
private Map<String,String> getModifiedFiles(Git git) throws GitAPIException, IOException {
private Map<String,String> getModifiedFiles(Git git,String branchName) throws GitAPIException, IOException {
Repository repository = git.getRepository();
git.fetch().call();
AbstractTreeIterator oldTreeParser = prepareTreeParser(repository, "HEAD");
AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/master");
AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/"+branchName);
Map<String, String> modifiedFiles = new HashMap<>();
try (DiffFormatter formatter = new DiffFormatter(System.out)) {

1
src/main/java/com/olympus/apollo/services/StorageProperties.java
View File

@@ -11,6 +11,7 @@ public class StorageProperties {
//private String location = ingestionRepositoryBasePath;
private String location = "/mnt/apollo_storage/documents";
//private String location = "C:\\repos\\olympus_ai\\Documents";
public String getLocation() {
return location;
}

2
src/main/resources/application.properties
View File

@@ -32,4 +32,4 @@ ingestion.repository.basepath=C:\\Users\\andrea.terzani\\dev\\Olympus
gitlab.token=
#gitlab.path=C:\\repos\\olympus_ai\\gitClone
gitlab.path=/mnt/apollo_storage/repository;
gitlab.path=/mnt/apollo_storage/repository