admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding code repository full BE implementation

Browse Source
This commit is contained in:
Mishra
2024-08-08 16:23:32 +05:30
parent 0c386d4d06
commit b90acc55e0
14 changed files with 653 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/olympus/apollo/ApolloApplication.java
View File

@@ -3,11 +3,13 @@ package com.olympus.apollo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.scheduling.annotation.EnableAsync;
import com.olympus.apollo.services.StorageProperties;
@SpringBootApplication
@EnableConfigurationProperties(StorageProperties.class)
@EnableAsync
public class ApolloApplication {
public static void main(String[] args) {

78
src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java Normal file
View File

@@ -0,0 +1,78 @@
package com.olympus.apollo.controllers.FeApi;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.olympus.apollo.dto.KSGitInfoDTO;
import com.olympus.apollo.dto.KSGitUploadDTO;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.KSGitIngestionInfoRepository;
import com.olympus.apollo.services.KSGitInfoService;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
@CrossOrigin(origins = "http://localhost:5173")
@RestController
@RequestMapping("/fe-api/ks_git_repos")
public class KSGitController {
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@Autowired
private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
@GetMapping("")
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
return result;
}
@PostMapping("/uploadRepo")
public ResponseEntity<String> handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) {
KSGitInfo ksGitInfo = new KSGitInfo();
ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName());
ksGitInfo.setBranch(ksGitUploadDTO.getBranch());
ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId());
ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", ksGitUploadDTO.getRepoName());
metadata.put("KsBranch", ksGitUploadDTO.getBranch());
metadata.put("KsRepoName", ksGitUploadDTO.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return ResponseEntity.ok("Upload successful");
}
}

103
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -1,14 +1,19 @@
package com.olympus.apollo.controllers;
import java.util.HashMap;
import java.util.List;
import com.olympus.apollo.dto.IngestionOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.olympus.apollo.models.KSIngestionInfo;
import com.olympus.apollo.dto.IngestionOutput;
import com.olympus.apollo.services.GitRepositoryIngestor;
import com.olympus.apollo.services.KSIngestor;
@@ -16,61 +21,63 @@ import com.olympus.apollo.services.KSIngestor;
@RestController
public class TestController {
@Autowired
KSIngestor ksIngestor;
@Autowired
private KSIngestor ksIngestor;
@Autowired
GitRepositoryIngestor gitRepositoryIngestor;
@Autowired
GitRepositoryIngestor gitRepositoryIngestor;
@GetMapping("test/ingestion_loop")
public IngestionOutput testIngestionLoop() {
return ksIngestor.ingestLoop();
}
private static final Logger logger = LoggerFactory.getLogger(TestController.class);
@GetMapping("test/ingest_document/{id}")
public IngestionOutput ingestDocumentById(@PathVariable String id) {
return ksIngestor.ingestDocumentById(id);
}
@GetMapping("test/ingestion_loop")
public IngestionOutput testIngestionLoop() {
return ksIngestor.ingestLoop();
}
@GetMapping("test/query_vector")
public List<String> testSimilaritySearch(@RequestParam String query, @RequestParam String type) {
return ksIngestor.testSimilaritySearch(query,type);
}
@GetMapping("test/ingest_document/{id}")
public IngestionOutput ingestDocumentById(@PathVariable String id) {
return ksIngestor.ingestDocumentById(id);
}
@GetMapping("test/delete")
public String deleteAllFromVectore(@RequestParam String query) {
ksIngestor.deleteAll("3-automated-test-framework---atf.md");
return "Deleted";
}
@GetMapping("test/query_vector")
public List<String> testSimilaritySearch(@RequestParam String query, @RequestParam String type) {
return ksIngestor.testSimilaritySearch(query, type);
}
@GetMapping("test/ingest_repo")
public String ingestRepo() {
try {
@GetMapping("test/delete")
public String deleteAllFromVectore(@RequestParam String query) {
ksIngestor.deleteAll("3-automated-test-framework---atf.md");
return "Deleted";
}
KSIngestionInfo ksIngestionInfo = new KSIngestionInfo();
@GetMapping("test/ingest_repo/{repoName}")
public ResponseEntity<String> ingestRepo(@PathVariable String repoName) {
try {
gitRepositoryIngestor.ingestGitRepository(repoName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicatioName","doo");
metadata.put("KsDoctype","sourcecode");
metadata.put("KsDoSource","GIT");
ksIngestionInfo.setMetadata(metadata);
ksIngestionInfo.setDefaultChunkSize(6000);
ksIngestionInfo.setMinChunkSize(200);
ksIngestionInfo.setMaxNumberOfChunks(10000);
ksIngestionInfo.setMinChunkSizeToEmbed(100);
logger.error("Error during ingestion start", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body("Error starting ingestion: " + e.getMessage());
}
}
String repoPath = "C:\\Users\\andrea.terzani\\dev\\DOO2_CLOUD";
gitRepositoryIngestor.ingestGitRepository(repoPath, ksIngestionInfo);
@GetMapping("test/check_ingestion_status/{repoName}")
public ResponseEntity<IngestionOutput> checkIngestionStatus(@PathVariable String repoName) {
try {
IngestionOutput ingestionOutput = gitRepositoryIngestor.checkIngestionStatus(repoName);
return ResponseEntity.ok(ingestionOutput);
} catch (Exception e) {
logger.error("Error checking ingestion status", e);
return "Ingested";
} catch (Exception e) {
return "Error";
}
}
IngestionOutput errorOutput = new IngestionOutput();
errorOutput.setStatus("ERROR");
errorOutput.setMessage("Error checking ingestion status: " + e.getMessage());
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorOutput);
}
}
}

22
src/main/java/com/olympus/apollo/dto/KSGitInfoDTO.java Normal file
View File

@@ -0,0 +1,22 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
import java.util.Date;
@Getter @Setter
public class KSGitInfoDTO {
private String repoName;
private String branch;
private String ingestionStatus;
private Date ingestionDate;
public KSGitInfoDTO(String repoName, String branch, String ingestionStatus, Date ingestionDate) {
this.repoName = repoName;
this.branch = branch;
this.ingestionStatus = ingestionStatus;
this.ingestionDate = ingestionDate;
}
}

19
src/main/java/com/olympus/apollo/dto/KSGitUploadDTO.java Normal file
View File

@@ -0,0 +1,19 @@
package com.olympus.apollo.dto;
import java.util.HashMap;
import lombok.Getter;
import lombok.Setter;
@Getter @Setter
public class KSGitUploadDTO {
private String repoName;
private String branch;
private String commitId;
private String repoPath;
private HashMap<String, String> metadata;
private int minChunkSizeToEmbed;
private int maxNumberOfChunks;
private int minChunkSize;
private int defaultChunkSize;
}

26
src/main/java/com/olympus/apollo/models/KSGitInfo.java Normal file
View File

@@ -0,0 +1,26 @@
package com.olympus.apollo.models;
import java.util.Date;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Getter;
import lombok.Setter;
@Document(collection = "ksgit_info")
@Getter
@Setter
public class KSGitInfo {
@Id
private String id;
private String repoName;
private String branch;
private String commitId;
private String repoPath;
private KSGitIngestionInfo ksGitIngestionInfo;
private String ingestionStatus;
private Date ingestionDate;
private String ingestionDateFormat;
}

21
src/main/java/com/olympus/apollo/models/KSGitIngestionInfo.java Normal file
View File

@@ -0,0 +1,21 @@
package com.olympus.apollo.models;
import java.util.HashMap;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Getter;
import lombok.Setter;
@Document(collection = "ksgit_ingestioninfo")
@Getter @Setter
public class KSGitIngestionInfo {
@Id
private String id;
private String type;
private HashMap<String, String> metadata;
private int minChunkSizeToEmbed;
private int maxNumberOfChunks;
private int minChunkSize;
private int defaultChunkSize;
}

15
src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java Normal file
View File

@@ -0,0 +1,15 @@
package com.olympus.apollo.repository;
import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.rest.core.annotation.RepositoryRestResource;
import org.springframework.stereotype.Repository;
import org.springframework.web.bind.annotation.CrossOrigin;
import com.olympus.apollo.models.KSGitInfo;
@Repository
public interface KSGitInfoRepository extends MongoRepository<KSGitInfo, String> {
Optional<KSGitInfo> findByRepoName(String repoName);
}

10
src/main/java/com/olympus/apollo/repository/KSGitIngestionInfoRepository.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.repository;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.stereotype.Repository;
import com.olympus.apollo.models.KSGitIngestionInfo;
@Repository
public interface KSGitIngestionInfoRepository extends MongoRepository<KSGitIngestionInfo, String> {
}

6
src/main/java/com/olympus/apollo/services/FileSystemStorageService.java
View File

@@ -7,6 +7,11 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Autowired;
@@ -110,5 +115,6 @@ public class FileSystemStorageService implements StorageService {
throw new StorageException("Could not initialize storage", e);
}
}
}

457
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -1,121 +1,398 @@
package com.olympus.apollo.services;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import com.olympus.apollo.models.KSIngestionInfo;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.olympus.apollo.dto.IngestionOutput;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.repository.KSGitInfoRepository;
@Service
public class GitRepositoryIngestor {
private final VectorStore vectorStore;
private final VectorStore vectorStore;
public GitRepositoryIngestor( VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
public void ingestGitRepository(String repoPath,KSIngestionInfo ingestionInfo) throws Exception {
try (Git git = Git.open(new File(repoPath))) {
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
List<Document> documents = new ArrayList<>();
@Async
public CompletableFuture<Void> ingestGitRepository(String repo) {
String repoPath = "C:\\Users\\vinayak.c.mishra\\dev\\olympus\\upload-dir\\" + repo + "\\";
logger.info("Repository path : " + repoPath);
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
}
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(),
ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(),
false);
List<Document> splitDocuments = splitter.split(documents);
vectorStore.add(splitDocuments);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("NEW".equals(ksGitInfo.getIngestionStatus())) {
ingestRepo(repoPath, ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
}
} else {
logger.info("OOPS: Document Not found");
}
return CompletableFuture.completedFuture(null);
}
private boolean isRelevantFile(String fileName) {
// Add more relevant file extensions as needed
return fileName.endsWith(".java");
}
private void ingestRepo(String repoPath, KSGitInfo ksGitInfo) {
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
private Map<String, String> extractMetadata(String fileName, String fileContent) {
Map<String, String> metadata = new HashMap<>();
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata : " + ingestionInfo.getMetadata());
ksGitInfoRepository.save(ksGitInfo);
if (fileName.endsWith(".java")) {
metadata.putAll(extractJavaMetadata(fileContent));
} else if (fileName.endsWith(".py")) {
metadata.putAll(extractPythonMetadata(fileContent));
} else if (fileName.endsWith(".js")) {
metadata.putAll(extractJavaScriptMetadata(fileContent));
}
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
return metadata;
}
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
private Map<String, String> extractJavaMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Simple regex to find class names (this is a basic implementation and might miss some cases)
Pattern classPattern = Pattern.compile("class\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
return metadata;
}
List<Document> documents = new ArrayList<>();
private Map<String, String> extractPythonMetadata(String fileContent) {
// Implement Python-specific metadata extraction
// This is a placeholder and should be implemented based on your needs
return new HashMap<>();
}
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
}
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(), false);
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents: " + splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded");
}
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
} catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e);
}
}
public IngestionOutput checkIngestionStatus(String repoName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repoName);
IngestionOutput ingestionOutput = new IngestionOutput();
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
ingestionOutput.setStatus(ksGitInfo.getIngestionStatus());
ingestionOutput.setMessage("Status Retrieved");
if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) {
ingestionOutput.getIngestedDocumentId().add(ksGitInfo.getId());
}
} else {
ingestionOutput.setStatus("ERROR");
ingestionOutput.setMessage("Document Not Found");
}
return ingestionOutput;
}
private boolean isRelevantFile(String fileName) {
// Add more relevant file extensions as needed
boolean response = false;
if (fileName.endsWith(".java")) {
response = true;
} else if (fileName.endsWith(".py")) {
response = true;
} else if (fileName.endsWith(".js")) {
response = true;
} else if (fileName.endsWith(".vue")) {
response = true;
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
response = true;
}
return response;
}
private Map<String, String> extractMetadata(String fileName, String fileContent) {
Map<String, String> metadata = new HashMap<>();
if (fileName.endsWith(".java")) {
metadata.putAll(extractJavaMetadata(fileContent));
} else if (fileName.endsWith(".py")) {
metadata.putAll(extractPythonMetadata(fileContent));
} else if (fileName.endsWith(".js")) {
metadata.putAll(extractJavaScriptMetadata(fileContent));
} else if (fileName.endsWith(".vue")) {
metadata.putAll(extractVueMetadata(fileContent));
} else if (fileName.endsWith(".groovy") || fileName.endsWith(".jenkins") || fileName.endsWith(".jenkinsfile")) {
metadata.putAll(extractGroovyMetadata(fileContent));
}
return metadata;
}
/*
* private Map<String, String> extractJavaMetadata(String fileContent) {
* Map<String, String> metadata = new HashMap<>(); // Simple regex to find class
* names (this is a basic implementation and might // miss some cases) Pattern
* classPattern = Pattern.compile("class\\s+(\\w+)"); Matcher classMatcher =
* classPattern.matcher(fileContent); List<String> classNames = new
* ArrayList<>(); while (classMatcher.find()) {
* classNames.add(classMatcher.group(1)); } metadata.put("classNames",
* String.join(",", classNames)); return metadata; }
*/
private Map<String, String> extractJavaMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract package name
Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);");
Matcher packageMatcher = packagePattern.matcher(fileContent);
if (packageMatcher.find()) {
metadata.put("packageName", packageMatcher.group(1));
}
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract method names
Pattern methodPattern = Pattern.compile("\\b(?:public|protected|private|static|\\s)\\s*\\w+\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.\\*]+);");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
return metadata;
}
private Map<String, String> extractPythonMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract function names
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+([\\w\\.]+)|\\bfrom\\s+([\\w\\.]+)\\s+import");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
if (importMatcher.group(1) != null) {
importStatements.add(importMatcher.group(1));
} else if (importMatcher.group(2) != null) {
importStatements.add(importMatcher.group(2));
}
}
metadata.put("importStatements", String.join(",", importStatements));
return metadata;
}
private Map<String, String> extractJavaScriptMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract function names
Pattern functionPattern = Pattern.compile("\\bfunction\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract import statements
Pattern importPattern = Pattern.compile("\\bimport\\s+[^;]+\\s+from\\s+['\"]([\\w\\.\\/-]+)['\"]");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
return metadata;
}
private Map<String, String> extractVueMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract component name
Pattern namePattern = Pattern.compile("<script>\\s*export\\s+default\\s+\\{\\s*name:\\s*['\"](\\w+)['\"]");
Matcher nameMatcher = namePattern.matcher(fileContent);
if (nameMatcher.find()) {
metadata.put("componentName", nameMatcher.group(1));
}
// Extract ref() usage
Pattern refPattern = Pattern.compile("\\bref\\(([^)]*)\\)");
Matcher refMatcher = refPattern.matcher(fileContent);
List<String> refs = new ArrayList<>();
while (refMatcher.find()) {
refs.add(refMatcher.group(1).replaceAll("[\"']", ""));
}
metadata.put("refs", String.join(",", refs));
// Check if template, script, and style tags are present
if (fileContent.contains("<template>")) {
metadata.put("hasTemplate", "true");
}
if (fileContent.contains("<script>")) {
metadata.put("hasScript", "true");
}
if (fileContent.contains("<style>")) {
metadata.put("hasStyle", "true");
}
return metadata;
}
private Map<String, String> extractGroovyMetadata(String fileContent) {
Map<String, String> metadata = new HashMap<>();
// Extract class names
Pattern classPattern = Pattern.compile("\\bclass\\s+(\\w+)");
Matcher classMatcher = classPattern.matcher(fileContent);
List<String> classNames = new ArrayList<>();
while (classMatcher.find()) {
classNames.add(classMatcher.group(1));
}
metadata.put("classNames", String.join(",", classNames));
// Extract method names
Pattern methodPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher methodMatcher = methodPattern.matcher(fileContent);
List<String> methodNames = new ArrayList<>();
while (methodMatcher.find()) {
methodNames.add(methodMatcher.group(1));
}
metadata.put("methodNames", String.join(",", methodNames));
// Extract import statements
Pattern importPattern = Pattern.compile("import\\s+([\\w\\.]+)");
Matcher importMatcher = importPattern.matcher(fileContent);
List<String> importStatements = new ArrayList<>();
while (importMatcher.find()) {
importStatements.add(importMatcher.group(1));
}
metadata.put("importStatements", String.join(",", importStatements));
// Extract script variables
Pattern variablePattern = Pattern.compile("\\bdef\\s+(\\w+)");
Matcher variableMatcher = variablePattern.matcher(fileContent);
List<String> scriptVariables = new ArrayList<>();
while (variableMatcher.find()) {
scriptVariables.add(variableMatcher.group(1));
}
metadata.put("scriptVariables", String.join(",", scriptVariables));
// Extract stages (for Jenkinsfiles)
Pattern stagePattern = Pattern.compile("\\bstage\\s*\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher stageMatcher = stagePattern.matcher(fileContent);
List<String> stageNames = new ArrayList<>();
while (stageMatcher.find()) {
stageNames.add(stageMatcher.group(1));
}
metadata.put("stageNames", String.join(",", stageNames));
// Extract function names (for Jenkinsfiles)
Pattern functionPattern = Pattern.compile("\\bdef\\s+(\\w+)\\s*\\(");
Matcher functionMatcher = functionPattern.matcher(fileContent);
List<String> functionNames = new ArrayList<>();
while (functionMatcher.find()) {
functionNames.add(functionMatcher.group(1));
}
metadata.put("functionNames", String.join(",", functionNames));
// Extract libraries/plugins (for Jenkinsfiles)
Pattern libraryPattern = Pattern.compile("library\\s+\\(\\s*['\"](.*?)['\"]\\s*\\)");
Matcher libraryMatcher = libraryPattern.matcher(fileContent);
List<String> libraries = new ArrayList<>();
while (libraryMatcher.find()) {
libraries.add(libraryMatcher.group(1));
}
metadata.put("libraries", String.join(",", libraries));
return metadata;
}
private Map<String, String> extractJavaScriptMetadata(String fileContent) {
// Implement JavaScript-specific metadata extraction
// This is a placeholder and should be implemented based on your needs
return new HashMap<>();
}
}

28
src/main/java/com/olympus/apollo/services/KSGitInfoService.java Normal file
View File

@@ -0,0 +1,28 @@
package com.olympus.apollo.services;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.olympus.apollo.dto.KSGitInfoDTO;
import com.olympus.apollo.repository.KSGitInfoRepository;
@Service
public class KSGitInfoService {
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
public List<KSGitInfoDTO> listGitInfo() {
return ksGitInfoRepository.findAll().stream()
.map(repo -> new KSGitInfoDTO(
repo.getRepoName(),
repo.getBranch(),
repo.getIngestionStatus(),
repo.getIngestionDate()
))
.collect(Collectors.toList());
}
}

4
src/main/java/com/olympus/apollo/services/StorageProperties.java
View File

@@ -7,8 +7,8 @@ public class StorageProperties {
/**
* Folder location for storing files
*/
private String location = "C:\\Users\\andrea.terzani\\dev\\olympus\\upload-dir";
private String location = "C:\\Users\\vinayak.c.mishra\\dev\\olympus\\upload-dir";
//private String location = "C:\\WindTre\\Repository";
public String getLocation() {
return location;
}

4
src/main/resources/application.properties
View File

@@ -24,6 +24,6 @@ spring.ai.vectorstore.mongodb.initialize-schema=false
spring.ai.openai.api-key=sk-proj-j3TFJ0h348DIzMrYYfyUT3BlbkFJjk4HMc8A2ux2Asg8Y7H1
#size filter
spring.servlet.multipart.max-file-size=10MB
spring.servlet.multipart.max-request-size=10MB
spring.servlet.multipart.max-file-size=5000000MB
spring.servlet.multipart.max-request-size=500000MB