diff --git a/apollo.code-workspace b/apollo.code-workspace new file mode 100644 index 0000000..6f4ce07 --- /dev/null +++ b/apollo.code-workspace @@ -0,0 +1,10 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../olympus-common" + } + ] +} \ No newline at end of file diff --git a/chroma-data/chroma.sqlite3 b/chroma-data/chroma.sqlite3 new file mode 100644 index 0000000..4022737 Binary files /dev/null and b/chroma-data/chroma.sqlite3 differ diff --git a/chroma-data/chroma.sqlite3.backup.0.5.5 b/chroma-data/chroma.sqlite3.backup.0.5.5 new file mode 100644 index 0000000..392543c Binary files /dev/null and b/chroma-data/chroma.sqlite3.backup.0.5.5 differ diff --git a/mvnw b/mvnw old mode 100644 new mode 100755 diff --git a/pom.xml b/pom.xml index 30d0872..c722408 100644 --- a/pom.xml +++ b/pom.xml @@ -88,11 +88,6 @@ spring-ai-starter-model-azure-openai - - org.springframework.ai - spring-ai-starter-vector-store-chroma - - org.springframework.ai spring-ai-tika-document-reader diff --git a/src/main/java/com/olympus/apollo/client/NewDocumentServiceClient.java b/src/main/java/com/olympus/apollo/client/NewDocumentServiceClient.java new file mode 100644 index 0000000..65f2789 --- /dev/null +++ b/src/main/java/com/olympus/apollo/client/NewDocumentServiceClient.java @@ -0,0 +1,146 @@ +package com.olympus.apollo.client; + +import com.olympus.apollo.dto.AgentSearchRequest; +import com.olympus.apollo.dto.AgentSearchResponse; +import com.olympus.apollo.dto.DeleteDocumentResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class NewDocumentServiceClient { + + private static final Logger logger = LoggerFactory.getLogger(NewDocumentServiceClient.class); + + @Value("${new-document-service.url}") + private String serviceUrl; + + private final RestTemplate restTemplate; + + public NewDocumentServiceClient() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(5000); // 5 seconds connection timeout + factory.setReadTimeout(60000); // 60 seconds read timeout + factory.setBufferRequestBody(false); // Don't buffer request body + + this.restTemplate = new RestTemplate(factory); + logger.info("RestTemplate initialized with timeouts - connect: 5s, read: 60s"); + } + + /** + * Delete all documents with the given KsDocumentId. + * + * This endpoint will: + * 1. Search for all documents with the specified KsDocumentId + * 2. Delete all matching documents from the index + * 3. Return the count of deleted documents + * + * @param ksDocumentId The KsDocumentId to delete + * @param project Project name for index isolation (default: "default") + * @param esUrl Elasticsearch URL (default: "http://localhost:9200") + * @return DeleteDocumentResponse with deletion details + */ + public ResponseEntity deleteDocumentByKsDocumentId( + String ksDocumentId, String project, String esUrl) { + + try { + // Build the URL with query parameters + UriComponentsBuilder uriBuilder = UriComponentsBuilder + .fromHttpUrl(serviceUrl) + .path("/api/documents/by-ks-document-id/{ks_document_id}") + .queryParam("project", project != null ? project : "default"); + + if (esUrl != null && !esUrl.isEmpty()) { + uriBuilder.queryParam("es_url", esUrl); + } + + String url = uriBuilder.buildAndExpand(ksDocumentId).toUriString(); + + logger.info("Deleting document with KsDocumentId: {} from search index at URL: {}", ksDocumentId, url); + + // Execute DELETE request + ResponseEntity response = restTemplate.exchange( + url, + HttpMethod.DELETE, + null, + DeleteDocumentResponse.class + ); + + logger.info("Delete request completed with status: {}", response.getStatusCode()); + return response; + + } catch (Exception e) { + logger.error("Error deleting document with KsDocumentId {}: {}", ksDocumentId, e.getMessage(), e); + throw e; + } + } + + /** + * Unified search endpoint for AI agents. + * + * Supports multiple search types: + * - semantic: Neural/vector similarity search + * - keyword: Exact term matching (AND operator) + * - fulltext: Analyzed text matching with fuzziness + * - hybrid: Combines semantic similarity with entity matching + * + * @param searchRequest The search request containing query, project, and search parameters + * @param esUrl Elasticsearch URL (default: "http://localhost:9200") + * @return AgentSearchResponse with search results and formatted context + */ + public ResponseEntity agentSearch( + AgentSearchRequest searchRequest, String esUrl) { + + try { + // Build the URL with query parameters + UriComponentsBuilder uriBuilder = UriComponentsBuilder + .fromHttpUrl(serviceUrl) + .path("/api/agent/search"); + + + String url = uriBuilder.toUriString(); + + logger.info("Performing agent search for project: {} with query: {} using search type: {}", + searchRequest.getProject(), searchRequest.getQuery(), searchRequest.getSearchType()); + + // Set up headers + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + // Create request entity + HttpEntity requestEntity = new HttpEntity<>(searchRequest, headers); + + // Execute POST request + logger.info("Executing POST request to: {}", url); + ResponseEntity response = restTemplate.exchange( + url, + HttpMethod.POST, + requestEntity, + AgentSearchResponse.class + ); + + logger.info("Agent search HTTP call completed with status: {}", response.getStatusCode()); + + if (response.getBody() != null) { + logger.info("Response body received with {} results", response.getBody().getTotalResults()); + } + + logger.info("Returning response from client"); + return response; + + } catch (Exception e) { + logger.error("Error performing agent search for project {}: {}", + searchRequest.getProject(), e.getMessage(), e); + throw e; + } + } +} diff --git a/src/main/java/com/olympus/apollo/config/OpenApiConfig.java b/src/main/java/com/olympus/apollo/config/OpenApiConfig.java new file mode 100644 index 0000000..0c63ab2 --- /dev/null +++ b/src/main/java/com/olympus/apollo/config/OpenApiConfig.java @@ -0,0 +1,32 @@ +package com.olympus.apollo.config; + +import io.swagger.v3.oas.models.Components; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.security.SecurityRequirement; +import io.swagger.v3.oas.models.security.SecurityScheme; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenApiConfig { + + @Bean + public OpenAPI customOpenAPI() { + final String securitySchemeName = "bearerAuth"; + + return new OpenAPI() + .info(new Info() + .title("Apollo API") + .version("1.0") + .description("Apollo Service API Documentation")) + .addSecurityItem(new SecurityRequirement().addList(securitySchemeName)) + .components(new Components() + .addSecuritySchemes(securitySchemeName, new SecurityScheme() + .name(securitySchemeName) + .type(SecurityScheme.Type.HTTP) + .scheme("bearer") + .bearerFormat("JWT") + .description("Enter JWT token obtained from /api/auth/login endpoint"))); + } +} diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java index cb83b40..c0f7677 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java @@ -3,10 +3,13 @@ package com.olympus.apollo.controllers.FeApi; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.*; +import com.olympus.apollo.dto.KnowledgeSystemNode; import com.olympus.model.apollo.KSDocument; import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.security.entity.User; import com.olympus.apollo.services.KSDocumentService; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; @@ -45,6 +48,23 @@ public class KsDocumentController { return ksDocumentService.downloadKSDocument(doc); -} + } + + /** + * Deletes a document by its ID + * The service method will be extended to handle search index cancellation + * @param id The ID of the document to delete + * @return ResponseEntity indicating success or failure + */ + @DeleteMapping("/{id}") + public ResponseEntity deleteDocument(@PathVariable String id) { + boolean deleted = ksDocumentService.deleteDocument(id); + + if (deleted) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.notFound().build(); + } + } } diff --git a/src/main/java/com/olympus/apollo/controllers/KSFileController.java b/src/main/java/com/olympus/apollo/controllers/KSFileController.java index 39b78b3..9890412 100644 --- a/src/main/java/com/olympus/apollo/controllers/KSFileController.java +++ b/src/main/java/com/olympus/apollo/controllers/KSFileController.java @@ -15,6 +15,7 @@ import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.ExceptionHandler; @@ -52,13 +53,15 @@ public class KSFileController { private static final Logger logger = LoggerFactory.getLogger(KSFileController.class); - private String document_path="/mnt/apollo_storage/documents"; + @Value("${document.storage.path}") + private String document_path; @PostMapping("/upload") public ResponseEntity handleFileUpload( @RequestParam("file") MultipartFile file, - @ModelAttribute FileUploadDTO fileUploadDTO + @ModelAttribute FileUploadDTO fileUploadDTO, + @RequestParam(value = "version", required = false) String version ) { try (InputStream inputStream = file.getInputStream()) { @@ -88,7 +91,13 @@ public class KSFileController { ksDocument.setFileName(file.getOriginalFilename()); ksDocument.setName(file.getOriginalFilename()); ksDocument.setDescription(fileUploadDTO.getDescription()); - ksDocument.setIngestionStatus("INGESTION_QUEUE"); + + if (version != null && version.contains("v2")) { + ksDocument.setIngestionStatus("IGNORE_INGESTION"); + ksDocument.setIngestionStatusV2("INGESTION_QUEUE"); + } else { + ksDocument.setIngestionStatus("INGESTION_QUEUE"); + } ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); Date now = new Date(); @@ -103,6 +112,14 @@ public class KSFileController { metadata.put("KsDocSource", fileUploadDTO.getKsDocSource()); metadata.put("KsFileSource", file.getOriginalFilename()); metadata.put("KsProjectName", fileUploadDTO.getKsProjectName()); + + logger.info("[UPLOAD] ksKnowledgePath received: {}", fileUploadDTO.getKsKnowledgePath()); + if (fileUploadDTO.getKsKnowledgePath() != null && !fileUploadDTO.getKsKnowledgePath().isEmpty()) { + metadata.put("ksKnowledgePath", fileUploadDTO.getKsKnowledgePath()); + logger.info("[UPLOAD] Added ksKnowledgePath to metadata: {}", fileUploadDTO.getKsKnowledgePath()); + } else { + logger.warn("[UPLOAD] ksKnowledgePath is null or empty"); + } ksIngestionInfo.setMetadata(metadata); ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize()); diff --git a/src/main/java/com/olympus/apollo/controllers/KnowledgeTreeController.java b/src/main/java/com/olympus/apollo/controllers/KnowledgeTreeController.java new file mode 100644 index 0000000..0d34e61 --- /dev/null +++ b/src/main/java/com/olympus/apollo/controllers/KnowledgeTreeController.java @@ -0,0 +1,393 @@ +package com.olympus.apollo.controllers; + +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ModelAttribute; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.olympus.apollo.client.NewDocumentServiceClient; +import com.olympus.apollo.dto.AgentSearchRequest; +import com.olympus.apollo.dto.AgentSearchResponse; +import com.olympus.apollo.dto.KnowledgeSystemNode; +import com.olympus.apollo.exception.StorageException; +import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.repository.KSVideoRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.apollo.services.KSDocumentService; +import com.olympus.dto.FileUploadDTO; +import com.olympus.dto.VideoUploadDTO; +import com.olympus.model.apollo.KSDocument; +import com.olympus.model.apollo.KSIngestionInfo; +import com.olympus.model.apollo.KSVideo; +import com.olympus.model.apollo.KSVideoIngestionInfo; + +/** + * KnowledgeTree Controller - V2 API + * This controller manages the new version of Knowledge Tree operations including + * file uploads and video uploads for the new application flow. + * Legacy endpoints remain in KSFileController and VideoController for backward compatibility. + */ +@RestController +@RequestMapping("/api/v2/knowledgetree") +public class KnowledgeTreeController { + + @Autowired + private KSDocumentRepository ksDocumentRepository; + + @Autowired + private KSVideoRepository ksVideoRepository; + + @Autowired + private KSDocumentService ksDocumentService; + + @Autowired + private NewDocumentServiceClient newDocumentServiceClient; + + @Value("${document.storage.path}") + private String documentStoragePath; + + @Value("${video.storage.path}") + private String videoStoragePath; + + private static final Logger logger = LoggerFactory.getLogger(KnowledgeTreeController.class); + + /** + * Upload a file to the knowledge tree (V2) + * Enhanced version with improved metadata handling and knowledge path support + * + * @param file The file to upload + * @param fileUploadDTO Upload metadata + * @return ResponseEntity containing the created KSDocument + */ + @PostMapping("/files/upload") + public ResponseEntity handleFileUpload( + @RequestParam("file") MultipartFile file, + @ModelAttribute FileUploadDTO fileUploadDTO) { + + logger.info("[V2-UPLOAD] Starting file upload for: {}", file.getOriginalFilename()); + logger.info("[V2-UPLOAD] Project: {}, Knowledge Path: {}", + fileUploadDTO.getKsProjectName(), + fileUploadDTO.getKsKnowledgePath()); + + try (InputStream inputStream = file.getInputStream()) { + // Validate file + if (file.isEmpty()) { + logger.error("[V2-UPLOAD] File is empty: {}", file.getOriginalFilename()); + throw new StorageException("Failed to store empty file."); + } + + // Create storage directory + Path folderPath = Paths.get(documentStoragePath) + .resolve(fileUploadDTO.getKsProjectName()) + .normalize() + .toAbsolutePath(); + + Files.createDirectories(folderPath); + logger.info("[V2-UPLOAD] Created folder: {}", folderPath); + + // Store file + String filePath = folderPath.resolve(file.getOriginalFilename()).toString(); + try (OutputStream outputStream = new FileOutputStream(filePath)) { + byte[] buffer = new byte[8 * 1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + logger.info("[V2-UPLOAD] File stored successfully at: {}", filePath); + } + + // Create KSDocument entity with V2 ingestion status + KSDocument ksDocument = new KSDocument(); + ksDocument.setFilePath(filePath); + ksDocument.setFileName(file.getOriginalFilename()); + ksDocument.setName(file.getOriginalFilename()); + ksDocument.setDescription(fileUploadDTO.getDescription()); + + // V2 uses new ingestion flow + ksDocument.setIngestionStatus("IGNORE_INGESTION"); + ksDocument.setIngestionStatusV2("INGESTION_QUEUE"); + + Date now = new Date(); + ksDocument.setIngestionDate(now); + ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(now)); + + // Build ingestion info with enhanced metadata + KSIngestionInfo ksIngestionInfo = new KSIngestionInfo(); + ksIngestionInfo.setType(fileUploadDTO.getType()); + + HashMap metadata = new HashMap<>(); + metadata.put("KsApplicationName", fileUploadDTO.getKsApplicationName()); + metadata.put("KsDoctype", fileUploadDTO.getKsDocType()); + metadata.put("KsDocSource", fileUploadDTO.getKsDocSource()); + metadata.put("KsFileSource", file.getOriginalFilename()); + metadata.put("KsProjectName", fileUploadDTO.getKsProjectName()); + + // Knowledge path support for V2 + if (fileUploadDTO.getKsKnowledgePath() != null && !fileUploadDTO.getKsKnowledgePath().isEmpty()) { + metadata.put("ksKnowledgePath", fileUploadDTO.getKsKnowledgePath()); + logger.info("[V2-UPLOAD] Knowledge path added: {}", fileUploadDTO.getKsKnowledgePath()); + } else { + logger.warn("[V2-UPLOAD] No knowledge path provided"); + } + + ksIngestionInfo.setMetadata(metadata); + ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize()); + ksIngestionInfo.setMinChunkSize(fileUploadDTO.getMinChunkSize()); + ksIngestionInfo.setMaxNumberOfChunks(fileUploadDTO.getMaxNumberOfChunks()); + ksIngestionInfo.setMinChunkSizeToEmbed(fileUploadDTO.getMinChunkSizeToEmbed()); + + ksDocument.setIngestionInfo(ksIngestionInfo); + ksDocumentRepository.save(ksDocument); + + logger.info("[V2-UPLOAD] Document saved successfully with ID: {}", ksDocument.getId()); + return ResponseEntity.ok(ksDocument); + + } catch (StorageException e) { + logger.error("[V2-UPLOAD] Storage exception: {}", e.getMessage(), e); + return ResponseEntity.status(400).body(null); + } catch (Exception e) { + logger.error("[V2-UPLOAD] Unexpected error during file upload: {}", e.getMessage(), e); + return ResponseEntity.status(500).body(null); + } + } + + /** + * Upload a video to the knowledge tree (V2) + * Enhanced version with improved metadata handling + * + * @param file The video file to upload + * @param videoUploadDTO Upload metadata + * @return ResponseEntity containing the created KSVideo + */ + @PostMapping("/videos/upload") + public ResponseEntity handleVideoUpload( + @RequestParam("file") MultipartFile file, + @ModelAttribute VideoUploadDTO videoUploadDTO) { + + logger.info("[V2-VIDEO-UPLOAD] Starting video upload for: {}", file.getOriginalFilename()); + logger.info("[V2-VIDEO-UPLOAD] Project: {}, Video Group: {}", + videoUploadDTO.getKsProjectName(), + videoUploadDTO.getKsVideoGroupId()); + + try (InputStream inputStream = file.getInputStream()) { + // Validate file + if (file.isEmpty()) { + logger.error("[V2-VIDEO-UPLOAD] File is empty: {}", file.getOriginalFilename()); + throw new StorageException("Failed to store empty file."); + } + + // Create storage directory + Path folderPath = Paths.get(videoStoragePath) + .resolve(videoUploadDTO.getKsProjectName()) + .normalize() + .toAbsolutePath(); + + Files.createDirectories(folderPath); + logger.info("[V2-VIDEO-UPLOAD] Created folder: {}", folderPath); + + // Store video file + String filePath = folderPath.resolve(file.getOriginalFilename()).toString(); + try (OutputStream outputStream = new FileOutputStream(filePath)) { + byte[] buffer = new byte[8 * 1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + logger.info("[V2-VIDEO-UPLOAD] Video stored successfully at: {}", filePath); + } + + // Create KSVideo entity + KSVideo ksVideo = new KSVideo(); + ksVideo.setFilePath(filePath); + ksVideo.setFileName(file.getOriginalFilename()); + ksVideo.setName(file.getOriginalFilename()); + ksVideo.setDescription(videoUploadDTO.getDescription()); + ksVideo.setIngestionStatus("IGNORE_INGESTION"); + ksVideo.setIngestionStatusV2("INGESTION_QUEUE"); + Date now = new Date(); + ksVideo.setIngestionDate(now); + ksVideo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(now)); + + // Build video ingestion info + KSVideoIngestionInfo ksVideoIngestionInfo = new KSVideoIngestionInfo(); + ksVideoIngestionInfo.setType(videoUploadDTO.getType()); + + HashMap metadata = new HashMap<>(); + metadata.put("KsApplicationName", videoUploadDTO.getKsApplicationName()); + metadata.put("KsDoctype", videoUploadDTO.getKsDocType()); + metadata.put("KsDocSource", videoUploadDTO.getKsDocSource()); + metadata.put("KsFileSource", file.getOriginalFilename()); + metadata.put("KsVideoGroupId", videoUploadDTO.getKsVideoGroupId()); + metadata.put("KsProjectName", videoUploadDTO.getKsProjectName()); + + // Knowledge path support for V2 videos + if (videoUploadDTO.getKsKnowledgePath() != null && !videoUploadDTO.getKsKnowledgePath().isEmpty()) { + metadata.put("ksKnowledgePath", videoUploadDTO.getKsKnowledgePath()); + logger.info("[V2-VIDEO-UPLOAD] Knowledge path added: {}", videoUploadDTO.getKsKnowledgePath()); + } else { + logger.warn("[V2-VIDEO-UPLOAD] No knowledge path provided"); + } + + ksVideoIngestionInfo.setMetadata(metadata); + ksVideoIngestionInfo.setNumberOfChunkToEmbed(videoUploadDTO.getNumberOfChunkToEmbed()); + ksVideoIngestionInfo.setChunkDurationInSeconds(videoUploadDTO.getChunkDurationInSeconds()); + ksVideoIngestionInfo.setLanguage(videoUploadDTO.getLanguage()); + + ksVideo.setIngestionInfo(ksVideoIngestionInfo); + ksVideoRepository.save(ksVideo); + + logger.info("[V2-VIDEO-UPLOAD] Video saved successfully with ID: {}", ksVideo.getId()); + return ResponseEntity.ok(ksVideo); + + } catch (StorageException e) { + logger.error("[V2-VIDEO-UPLOAD] Storage exception: {}", e.getMessage(), e); + return ResponseEntity.status(400).body(null); + } catch (Exception e) { + logger.error("[V2-VIDEO-UPLOAD] Unexpected error during video upload: {}", e.getMessage(), e); + return ResponseEntity.status(500).body(null); + } + } + + /** + * Returns a filesystem-like tree structure based on ksKnowledgePath metadata + * for the current user's selected project + * @return KnowledgeSystemNode representing the root of the tree + */ + @GetMapping("/knowledge-tree") + public ResponseEntity getKnowledgeTree() { + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + String projectName = principal.getSelectedProject().getInternal_name(); + + KnowledgeSystemNode tree = ksDocumentService.buildKnowledgePathTree(projectName); + + if (tree == null) { + return ResponseEntity.noContent().build(); + } + + return ResponseEntity.ok(tree); + } + + /** + * Returns a filesystem-like tree structure based on ksKnowledgePath metadata + * for a specific project + * @param projectName The name of the project + * @return KnowledgeSystemNode representing the root of the tree + */ + @GetMapping("/knowledge-tree/{projectName}") + public ResponseEntity getKnowledgeTreeByProject(@PathVariable String projectName) { + KnowledgeSystemNode tree = ksDocumentService.buildKnowledgePathTree(projectName); + + if (tree == null) { + return ResponseEntity.noContent().build(); + } + + return ResponseEntity.ok(tree); + } + + + + /** + * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections + * for the current user's selected project + * + * @return Map where keys are metadata field names and values are sets of all possible values for each field + */ + @GetMapping("/metadata/all-values") + public ResponseEntity>> getAllMetadataValues() { + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + String projectName = principal.getSelectedProject().getInternal_name(); + + Map> allMetadata = ksDocumentService.getAllMetadataValues(projectName); + + if (allMetadata == null || allMetadata.isEmpty()) { + return ResponseEntity.noContent().build(); + } + + return ResponseEntity.ok(allMetadata); + } + + /** + * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections + * for a specific project + * + * @param projectName The name of the project + * @return Map where keys are metadata field names and values are sets of all possible values for each field + */ + @GetMapping("/metadata/all-values/{projectName}") + public ResponseEntity>> getAllMetadataValuesByProject(@PathVariable String projectName) { + Map> allMetadata = ksDocumentService.getAllMetadataValues(projectName); + + if (allMetadata == null || allMetadata.isEmpty()) { + return ResponseEntity.noContent().build(); + } + + return ResponseEntity.ok(allMetadata); + } + + /** + * Agent search endpoint - Unified search for AI agents + * Supports multiple search types: semantic, keyword, fulltext, and hybrid + * + * @param searchRequest The search request containing query, project, and search parameters + * @param esUrl Optional Elasticsearch URL (defaults to service configuration) + * @return ResponseEntity containing AgentSearchResponse with search results and formatted context + */ + @PostMapping("/agent-search") + public ResponseEntity agentSearch( + @RequestBody AgentSearchRequest searchRequest) { + + logger.info("[AGENT-SEARCH] Received search request for project: {}, query: {}, type: {}", + searchRequest.getProject(), + searchRequest.getQuery(), + searchRequest.getSearchType()); + + try { + logger.info("[AGENT-SEARCH] Calling document service client..."); + ResponseEntity response = + newDocumentServiceClient.agentSearch(searchRequest, null); + + logger.info("[AGENT-SEARCH] Client call completed with status: {}", response.getStatusCode()); + + AgentSearchResponse body = response.getBody(); + if (body != null) { + logger.info("[AGENT-SEARCH] Search completed successfully. Results: {}", + body.getTotalResults()); + } + + logger.info("[AGENT-SEARCH] Creating new ResponseEntity with body"); + // Create a new ResponseEntity to avoid serialization issues with the proxied response + return ResponseEntity + .status(response.getStatusCode()) + .headers(response.getHeaders()) + .body(body); + + } catch (Exception e) { + logger.error("[AGENT-SEARCH] Error performing agent search: {}", e.getMessage(), e); + return ResponseEntity.status(500).body(null); + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/olympus/apollo/controllers/SearchDocController.java b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java index 0b74a09..130fe92 100644 --- a/src/main/java/com/olympus/apollo/controllers/SearchDocController.java +++ b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java @@ -1,14 +1,9 @@ package com.olympus.apollo.controllers; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.slf4j.LoggerFactory; -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.ai.vectorstore.SearchRequest.Builder; -import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -22,34 +17,12 @@ import ch.qos.logback.classic.Logger; @RequestMapping("/api/vsearch") public class SearchDocController { - @Autowired - private VectorStore vectorStore; - Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class); @PostMapping("/doc_search") public List vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) { - - Builder request_builder = SearchRequest.builder() - .query(vectorSearchRequest.getQuery()) - .topK(vectorSearchRequest.getTopK()) - .similarityThreshold(vectorSearchRequest.getThreshold()); - - if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){ - request_builder.filterExpression(vectorSearchRequest.getFilterQuery()); - logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery()); - } - - SearchRequest request = request_builder.build(); - List docs = this.vectorStore.similaritySearch(request); - - logger.info("Number of VDB retrieved documents: " + docs.size()); - - List results = new ArrayList(); - for (Document doc : docs) { - results.add(doc.getText()); - } - return results; - + // VectorStore operations removed + logger.info("Vector search requested for query: " + vectorSearchRequest.getQuery()); + return Collections.emptyList(); } } diff --git a/src/main/java/com/olympus/apollo/dto/AgentSearchRequest.java b/src/main/java/com/olympus/apollo/dto/AgentSearchRequest.java new file mode 100644 index 0000000..c435e2c --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/AgentSearchRequest.java @@ -0,0 +1,119 @@ +package com.olympus.apollo.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +/** + * Request model for agent search endpoint. + * Supports multiple search types: semantic, keyword, fulltext, or hybrid. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AgentSearchRequest { + + /** + * Project name (required) + */ + private String project; + + /** + * Search query text + */ + private String query; + + /** + * Type of search: semantic, keyword, fulltext, or hybrid + * Default: hybrid + */ + @JsonProperty("search_type") + @Builder.Default + private String searchType = "hybrid"; + + /** + * Number of results to return (1-50) + * Default: 5 + */ + @Builder.Default + private Integer k = 5; + + /** + * Maximum characters for formatted context (100-50000) + * Default: 4000 + */ + @JsonProperty("max_context_chars") + @Builder.Default + private Integer maxContextChars = 4000; + + /** + * Weight for vector similarity in hybrid search (0.0-1.0) + * Default: 0.7 + */ + @JsonProperty("vector_weight") + @Builder.Default + private Double vectorWeight = 0.7; + + /** + * Weight for entity matching in hybrid search (0.0-1.0) + * Default: 0.3 + */ + @JsonProperty("entity_weight") + @Builder.Default + private Double entityWeight = 0.3; + + /** + * Filter by application name + */ + @JsonProperty("KsApplicationName") + private String ksApplicationName; + + /** + * Filter by document type + */ + @JsonProperty("KsDoctype") + private String ksDoctype; + + /** + * Filter by document source + */ + @JsonProperty("KsDocSource") + private String ksDocSource; + + /** + * Filter by file source + */ + @JsonProperty("KsFileSource") + private String ksFileSource; + + /** + * Filter by document ID + */ + @JsonProperty("KsDocumentId") + private String ksDocumentId; + + /** + * Filter by project name in metadata + */ + @JsonProperty("KsProjectName") + private String ksProjectName; + + /** + * Filter by knowledge folder + */ + @JsonProperty("KsKnowledgePath") + private String ksKnowledgePath; + + /** + * Filter by configurable tags + * Example: {"scope": "finance", "requirement_id": ["ID12211"]} + */ + private Map tags; +} diff --git a/src/main/java/com/olympus/apollo/dto/AgentSearchResponse.java b/src/main/java/com/olympus/apollo/dto/AgentSearchResponse.java new file mode 100644 index 0000000..8f4ef49 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/AgentSearchResponse.java @@ -0,0 +1,142 @@ +package com.olympus.apollo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +/** + * Response model for agent search endpoint. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AgentSearchResponse { + + /** + * The original search query + */ + private String query; + + /** + * The project searched + */ + private String project; + + /** + * The search type used + */ + @JsonProperty("search_type") + private String searchType; + + /** + * Total number of results found + */ + @JsonProperty("total_results") + private Integer totalResults; + + /** + * List of search results + */ + private List results; + + /** + * Pre-formatted context for LLM consumption + */ + @JsonProperty("formatted_context") + private String formattedContext; + + /** + * Additional metadata about the search + */ + private Map metadata; + + /** + * Individual search result item + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class SearchResultItem { + + /** + * Content of the document chunk + */ + private String content; + + /** + * Relevance score + */ + private Double score; + + /** + * File name + */ + @JsonProperty("file_name") + private String fileName; + + /** + * Chunk index within the document + */ + @JsonProperty("chunk_index") + private Integer chunkIndex; + + /** + * Entities found in this chunk + */ + private List> entities; + + /** + * Tags associated with this result + */ + private Map tags; + + /** + * Knowledge path + */ + @JsonProperty("ks_knowledge_path") + private String ksKnowledgePath; + + /** + * Document type + */ + @JsonProperty("ks_doctype") + private String ksDoctype; + + /** + * Application name + */ + @JsonProperty("ks_application_name") + private String ksApplicationName; + + /** + * Document source + */ + @JsonProperty("ks_doc_source") + private String ksDocSource; + + /** + * File source + */ + @JsonProperty("ks_file_source") + private String ksFileSource; + + /** + * Document ID + */ + @JsonProperty("ks_document_id") + private String ksDocumentId; + + /** + * Project name + */ + @JsonProperty("ks_project_name") + private String ksProjectName; + } +} diff --git a/src/main/java/com/olympus/apollo/dto/DeleteDocumentResponse.java b/src/main/java/com/olympus/apollo/dto/DeleteDocumentResponse.java new file mode 100644 index 0000000..7b1a137 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/DeleteDocumentResponse.java @@ -0,0 +1,13 @@ +package com.olympus.apollo.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class DeleteDocumentResponse { + private String message; + private int deletedCount; + private String ksDocumentId; + private String project; +} diff --git a/src/main/java/com/olympus/apollo/dto/KnowledgeSystemNode.java b/src/main/java/com/olympus/apollo/dto/KnowledgeSystemNode.java new file mode 100644 index 0000000..3f1ca1d --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/KnowledgeSystemNode.java @@ -0,0 +1,44 @@ +package com.olympus.apollo.dto; + +import com.olympus.model.apollo.KSDocument; +import com.olympus.model.apollo.KSVideo; +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class KnowledgeSystemNode { + + private String name; + private String path; + private NodeType type; + private List children; + private String documentId; + private KSDocument document; + private String videoId; + private KSVideo video; + + public KnowledgeSystemNode() { + this.children = new ArrayList<>(); + } + + public KnowledgeSystemNode(String name, String path, NodeType type) { + this.name = name; + this.path = path; + this.type = type; + this.children = new ArrayList<>(); + } + + public void addChild(KnowledgeSystemNode child) { + this.children.add(child); + } + + public enum NodeType { + FOLDER, + FILE, + VIDEO_FILE + } +} diff --git a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java index 1f86602..e510757 100644 --- a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java +++ b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java @@ -6,7 +6,6 @@ import java.util.List; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; import org.springframework.data.rest.core.annotation.RepositoryRestResource; import org.springframework.web.bind.annotation.CrossOrigin; @@ -24,4 +23,10 @@ public interface KSDocumentRepository extends MongoRepository findByProjectName(String projectName, Sort sort); + @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0 }") + public List findAllByProjectName(String projectName); + + @Query(value = "{ 'ingestionInfo.metadata.KsProjectName': ?0 }", fields = "{ 'ingestionInfo.metadata': 1 }") + public List findAllMetadataByProjectName(String projectName); + } diff --git a/src/main/java/com/olympus/apollo/repository/KSVideoRepository.java b/src/main/java/com/olympus/apollo/repository/KSVideoRepository.java index ff77db8..fda26b2 100644 --- a/src/main/java/com/olympus/apollo/repository/KSVideoRepository.java +++ b/src/main/java/com/olympus/apollo/repository/KSVideoRepository.java @@ -34,4 +34,7 @@ public interface KSVideoRepository extends MongoRepository { }) List countVideosByGroupIds(List groupIds); + @Query(value = "{ 'ingestionInfo.metadata.KsProjectName': ?0 }", fields = "{ 'ingestionInfo.metadata': 1 }") + public List findAllMetadataByProjectName(String projectName); + } diff --git a/src/main/java/com/olympus/apollo/services/DeletionService.java b/src/main/java/com/olympus/apollo/services/DeletionService.java index 77fcbad..b3da600 100644 --- a/src/main/java/com/olympus/apollo/services/DeletionService.java +++ b/src/main/java/com/olympus/apollo/services/DeletionService.java @@ -1,21 +1,16 @@ package com.olympus.apollo.services; import java.util.Date; -import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import com.olympus.apollo.exception.vectorStoreMetaDetailsEmptyException; import com.olympus.apollo.repository.KSDocumentRepository; import com.olympus.apollo.repository.KSGitInfoRepository; import com.olympus.apollo.repository.KSGitIngestionInfoRepository; @@ -51,17 +46,10 @@ public class DeletionService { @Autowired private SimpMessagingTemplate simpMessagingTemplate; - @Autowired - private VectorStore vectorStore; - @Async("asyncTaskExecutor") public void deleteRecords(DeletionRequest deletionRequest) { try { - - String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'"; logger.info("Starting deletion"); - vectorStore.delete(rag_filter); - ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId()); logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()); } catch (Exception e) { @@ -78,37 +66,14 @@ public class DeletionService { ksDocumentRepository.save(ksDocument); logger.info("Starting deletion"); - - String rag_filter = "KsDocumentId=='" + deletionRequest.getKsDocumentId() + "'"; - - - SearchRequest searchRequest = SearchRequest.builder() - .query(" ") - .filterExpression(rag_filter) - .topK(Integer.MAX_VALUE) - .build(); - - List idsToDelete = vectorStore.similaritySearch(searchRequest) - .stream() - .map(Document::getId) - .toList(); - - logger.info("Found {} documents to delete for KsDocumentId: {}", idsToDelete.size(), deletionRequest.getKsDocumentId()); - - //Batch per eliminare i file con più richieste - final int DELETE_BATCH_SIZE = 500; - for (int i = 0; i < idsToDelete.size(); i += DELETE_BATCH_SIZE) { - int end = Math.min(i + DELETE_BATCH_SIZE, idsToDelete.size()); - List batch = idsToDelete.subList(i, end); - logger.info("Deleting batch from {} to {}", i, end); - vectorStore.delete(batch); - } - + + // VectorStore operations removed + ksDocument.setIngestionStatus("LOADED"); ksDocument.setIngestionDate(new Date()); ksDocumentRepository.save(ksDocument); - logger.info("KSDocument with id {} deleted from VectorStore successfully.", deletionRequest.getKsDocumentId()); + logger.info("KSDocument with id {} status updated successfully.", deletionRequest.getKsDocumentId()); } catch (Exception e) { logger.error("An error occurred while deleting records: ", e); @@ -119,16 +84,9 @@ public class DeletionService { @Async("asyncTaskExecutor") public void deleteVideoRecords(DeletionRequest deletionRequest) { try { - - String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'"; logger.info("Starting deletion"); - vectorStore.delete(rag_filter); - ksVideoRepository.deleteById(deletionRequest.getKsDocumentId()); logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()); - // }else{ - // logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()); - // } } catch (Exception e) { logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName()); throw new RuntimeException("An error occurred while deleting records", e); @@ -138,26 +96,20 @@ public class DeletionService { @Async("asyncTaskExecutor") public void deleteVideoRecordsOnlyFromVectorStore(DeletionRequest deletionRequest) { try { - KSVideo ksVideo = ksVideoRepository.findById(deletionRequest.getKsDocumentId()).get(); ksVideo.setIngestionStatus("DELETING"); ksVideoRepository.save(ksVideo); - String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'"; - logger.info("Starting deletion"); - vectorStore.delete(rag_filter); - - //elimino dal vectorStore ma mantengo il record + + // VectorStore operations removed + ksVideo.setIngestionStatus("LOADED"); Date now = new Date(); ksVideo.setIngestionDate(now); ksVideoRepository.save(ksVideo); - logger.info("KSVideo with id {} deleted from VectorStore successfully.", deletionRequest.getKsDocumentId()); - // }else{ - // logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()); - // } + logger.info("KSVideo with id {} status updated successfully.", deletionRequest.getKsDocumentId()); } catch (Exception e) { logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName()); throw new RuntimeException("An error occurred while deleting records", e); @@ -179,7 +131,6 @@ public class DeletionService { boolean KSGitInfoExists = ksGitInfoId != null && !ksGitInfoId.isEmpty() && ksGitInfoRepository.existsById(ksGitInfoId); boolean KSGitIngestionInfoExists = ksGitIngestionInfoId != null && !ksGitIngestionInfoId.isEmpty() && ksGitIngestionInfoRepository.existsById(ksGitIngestionInfoId); - boolean vectorStoreGitDetailsExists = applicationName != null && ksDocSource != null && ksFileSource != null && ksDoctype != null && ksBranch != null; logger.info("KSGitInfo with id {} exists: {}", ksGitInfoId,KSGitInfoExists); logger.info("KSGitIngestionInfo with id {} exists: {}", ksGitIngestionInfoId,KSGitIngestionInfoExists); @@ -199,12 +150,8 @@ public class DeletionService { String ingestionStatus = ksGitInfo.getIngestionStatus(); logger.info("Ingestion Status is {}.", ingestionStatus); - List vectorStoreMetadataDetails = null; /*vectorStoreGitDetailsExists - ? vectorStoreRepository.findGitVectorByMetadata(ksDoctype,ksDocSource, ksFileSource, applicationName, ksBranch) - : List.of();*/ - if (KSGitInfoExists && KSGitIngestionInfoExists) { - deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,vectorStoreMetadataDetails,applicationName); + deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,applicationName); String message = applicationName + " With Branch " + ksBranch + " records removed successfully having KSGitInfo with id "+ksGitInfoId; logger.info(message); @@ -224,13 +171,6 @@ public class DeletionService { String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitIngestionInfo with id "+ksGitIngestionInfoId+" does not exist."; logger.error(message); - resultDTO.setSuccess(false); - resultDTO.setMessage(message); - simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); - } else if (vectorStoreMetadataDetails.isEmpty()) { - String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to No VectorStore Data available"; - logger.error(message); - resultDTO.setSuccess(false); resultDTO.setMessage(message); simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); @@ -249,32 +189,21 @@ public class DeletionService { }); } - private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,List vectorStoreMetaDetails,String applicationName){ + private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,String applicationName){ try{ switch (ingestionStatus){ case "INGESTION-ERROR": case "INGESTION-IN-PROGRESS": case "INGESTED": - deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName); - deleteVectorStores(vectorStoreMetaDetails,applicationName); - break; case "REPO-NEW": case "REPO-CLONE-IN-PROGRESS": case "REPO-CLONE-COMPLETED": case "REPO-CLONE-FAILED": - if (vectorStoreMetaDetails.isEmpty()) { - deleteGitInfoAndIngestionInfo(ksGitInfoId, ksGitIngestionInfoId, applicationName); - }else { - // Throw a custom exception if vectorStoreMetaDetails is not empty - throw new vectorStoreMetaDetailsEmptyException("VectorStoreMetaDetails is not empty for application name "+applicationName+" branch "+ksBranch+" and ingestion status is " + ingestionStatus); - } + deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName); break; default: logger.warn("Unknown ingestion status: {}", ingestionStatus); } - } catch (vectorStoreMetaDetailsEmptyException e){ - logger.error("vectorStoreMetaDetailsEmptyException occurred: ", e); - throw e; } catch (Exception e){ logger.error("An error occurred while deleting records based on ingestion status: ", e); throw new RuntimeException("An error occurred while deleting records based on ingestion status", e); @@ -293,45 +222,18 @@ public class DeletionService { } } - private void deleteVectorStores(List vectorStoreMetadataDetails, String applicationName){ - if(!vectorStoreMetadataDetails.isEmpty()){ - - /* for (VectorStore store : vectorStoreMetadataDetails) { - String storeId=store.getId(); - vectorStoreRepository.deleteById(storeId); - logger.info("VectorStore with id {} deleted successfully.", applicationName, storeId); - }*/ - } - } - - - @Async("asyncTaskExecutor") public void deleteTextRecords(String id) { try { boolean KSTextExists = ksTextsRepository.existsById(id); - - /* - List vectorStoreMetadataDetails = vectorStoreRepository.findByKsInternalMainEntityId(id); - - if (KSTextExists && !vectorStoreMetadataDetails.isEmpty()) { - for (VectorStore store : vectorStoreMetadataDetails) { - vectorStoreRepository.deleteById(store.getId()); - logger.info("VectorStore with id {} deleted successfully.", store.getId()+" "); - } - + + if (KSTextExists) { ksTextsRepository.deleteById(id); - logger.info("KSDocument with id {} deleted successfully.", id+" "); - - + logger.info("KSDocument with id {} deleted successfully.", id); logger.info("All records deleted successfully."); } else { - if (!KSTextExists) { - logger.warn("KSDocument with id {} does not exist.", id+" "); - } else if (vectorStoreMetadataDetails.isEmpty()) { - logger.warn("No VectorStore Data available",Thread.currentThread().getName()); - } - }*/ + logger.warn("KSDocument with id {} does not exist.", id); + } } catch (Exception e) { logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName()); throw new RuntimeException("An error occurred while deleting records", e); diff --git a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java index 4875d56..b63aea6 100644 --- a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +++ b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java @@ -15,7 +15,6 @@ import java.util.regex.Pattern; import com.olympus.dto.ResultDTO; import com.olympus.apollo.exception.BranchCheckoutException; -import com.olympus.apollo.repository.VectorStoreRepository; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Repository; @@ -25,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.transformer.splitter.TokenTextSplitter; -import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.messaging.simp.SimpMessagingTemplate; @@ -40,27 +38,18 @@ import com.olympus.apollo.repository.KSGitInfoRepository; @Service public class GitRepositoryIngestor { - private final VectorStore vectorStore; - @Value("${gitlab.path}") private String basePath; @Autowired private KSGitInfoRepository ksGitInfoRepository; - @Autowired - private VectorStoreRepository vectorStoreRepository; - @Autowired private GitService gitService; @Autowired private SimpMessagingTemplate simpMessagingTemplate; - public GitRepositoryIngestor(VectorStore vectorStore) { - this.vectorStore = vectorStore; - } - Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class); @Async @@ -147,8 +136,8 @@ public class GitRepositoryIngestor { List splitDocuments = splitter.split(documents); logger.info("Number of documents to be embedded: {}", splitDocuments.size()); - vectorStore.add(splitDocuments); - logger.info("Documents embedded Successfully"); + // VectorStore operations removed + logger.info("Documents processed successfully"); } catch (IOException e) { ksGitInfo.setIngestionStatus("ERROR"); @@ -242,13 +231,7 @@ public class GitRepositoryIngestor { default: break; } - for (String fileToDelete : filePathsToDelete) { - Optional optionalDocument = vectorStoreRepository.findByKsapplicationNameKsBranchFilePath(repoName,branchName,fileToDelete); - if (optionalDocument.isPresent()) { - String vectorStoreId = optionalDocument.get().getId(); - vectorStoreRepository.deleteById(vectorStoreId); - } - } + // VectorStore operations removed for file deletions } gitService.checkOutRepository(repoName,branchName); String repoPath = basePath+ File.separator + repoName; @@ -308,8 +291,8 @@ public class GitRepositoryIngestor { List splitDocuments = splitter.split(documents); logger.info("Number of documents: " + splitDocuments.size()); - vectorStore.add(splitDocuments); - logger.info("Documents embedded"); + // VectorStore operations removed + logger.info("Documents processed"); } diff --git a/src/main/java/com/olympus/apollo/services/KSDocumentService.java b/src/main/java/com/olympus/apollo/services/KSDocumentService.java index 67a5d1d..22a6741 100644 --- a/src/main/java/com/olympus/apollo/services/KSDocumentService.java +++ b/src/main/java/com/olympus/apollo/services/KSDocumentService.java @@ -3,7 +3,15 @@ package com.olympus.apollo.services; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,9 +24,14 @@ import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; +import com.olympus.apollo.client.NewDocumentServiceClient; +import com.olympus.apollo.dto.DeleteDocumentResponse; +import com.olympus.apollo.dto.KnowledgeSystemNode; import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.repository.KSVideoRepository; import com.olympus.apollo.security.entity.User; import com.olympus.model.apollo.KSDocument; +import com.olympus.model.apollo.KSVideo; @Service public class KSDocumentService { @@ -27,6 +40,12 @@ public class KSDocumentService { @Autowired private KSDocumentRepository ksdocRepo; + + @Autowired + private KSVideoRepository ksVideoRepo; + + @Autowired + private NewDocumentServiceClient newDocumentServiceClient; public List findByProjectNameAndApplicationName() { User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); @@ -68,4 +87,360 @@ public class KSDocumentService { return ResponseEntity.internalServerError().build(); } } + + /** + * Builds a filesystem-like structure based on ksKnowledgePath metadata for a given project + * Documents and videos without ksKnowledgePath in metadata are placed in the root folder + * @param projectName The name of the project to build the tree for + * @return Root node of the filesystem tree + */ + public KnowledgeSystemNode buildKnowledgePathTree(String projectName) { + logger.info("Building knowledge path tree for project: " + projectName); + + try { + // Get all documents and videos for this project + List documents = ksdocRepo.findAllByProjectName(projectName); + List videos = ksVideoRepo.findByProjectName(projectName, Sort.by(Sort.Direction.DESC, "ingestionDate")); + + if ((documents == null || documents.isEmpty()) && (videos == null || videos.isEmpty())) { + logger.warn("No documents or videos found for project: " + projectName); + return null; + } + + // Create root node + KnowledgeSystemNode root = new KnowledgeSystemNode("root", "/", KnowledgeSystemNode.NodeType.FOLDER); + + // Build the tree structure + for (KSDocument doc : documents) { + String path = extractKnowledgePathFromMetadata(doc); + + // If no path in metadata, add document directly to root + if (path == null || path.trim().isEmpty()) { + String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName(); + if (fileName != null && !fileName.isEmpty()) { + KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.FILE); + fileNode.setDocumentId(doc.getId()); + fileNode.setDocument(doc); + root.addChild(fileNode); + } + continue; + } + + // Normalize path: remove leading/trailing slashes and split + path = path.trim(); + if (path.startsWith("/")) { + path = path.substring(1); + } + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + if (path.isEmpty()) { + // Empty path after normalization, add to root + String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName(); + if (fileName != null && !fileName.isEmpty()) { + KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.FILE); + fileNode.setDocumentId(doc.getId()); + fileNode.setDocument(doc); + root.addChild(fileNode); + } + continue; + } + + String[] parts = path.split("/"); + KnowledgeSystemNode currentNode = root; + StringBuilder currentPath = new StringBuilder(); + + // Navigate/create folder structure for the entire path + // (ksKnowledgePath represents the folder where the document should be placed) + for (int i = 0; i < parts.length; i++) { + String folderName = parts[i]; + currentPath.append("/").append(folderName); + + KnowledgeSystemNode childNode = findOrCreateChild(currentNode, folderName, + currentPath.toString(), KnowledgeSystemNode.NodeType.FOLDER); + currentNode = childNode; + } + + // Add the file node using the document's actual fileName + String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName(); + if (fileName != null && !fileName.isEmpty()) { + currentPath.append("/").append(fileName); + KnowledgeSystemNode fileNode = findOrCreateChild(currentNode, fileName, + currentPath.toString(), KnowledgeSystemNode.NodeType.FILE); + fileNode.setDocumentId(doc.getId()); + fileNode.setDocument(doc); + } + } + + // Process videos with the same logic as documents + if (videos != null) { + for (KSVideo video : videos) { + String path = extractKnowledgePathFromVideoMetadata(video); + + // If no path in metadata, add video directly to root + if (path == null || path.trim().isEmpty()) { + String fileName = video.getFileName() != null ? video.getFileName() : video.getName(); + if (fileName != null && !fileName.isEmpty()) { + KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.VIDEO_FILE); + fileNode.setVideoId(video.getId()); + fileNode.setVideo(video); + root.addChild(fileNode); + } + continue; + } + + // Normalize path: remove leading/trailing slashes and split + path = path.trim(); + if (path.startsWith("/")) { + path = path.substring(1); + } + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + if (path.isEmpty()) { + // Empty path after normalization, add to root + String fileName = video.getFileName() != null ? video.getFileName() : video.getName(); + if (fileName != null && !fileName.isEmpty()) { + KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.VIDEO_FILE); + fileNode.setVideoId(video.getId()); + fileNode.setVideo(video); + root.addChild(fileNode); + } + continue; + } + + String[] parts = path.split("/"); + KnowledgeSystemNode currentNode = root; + StringBuilder currentPath = new StringBuilder(); + + // Navigate/create folder structure for the entire path + for (int i = 0; i < parts.length; i++) { + String folderName = parts[i]; + currentPath.append("/").append(folderName); + + KnowledgeSystemNode childNode = findOrCreateChild(currentNode, folderName, + currentPath.toString(), KnowledgeSystemNode.NodeType.FOLDER); + currentNode = childNode; + } + + // Add the video file node using the video's actual fileName + String fileName = video.getFileName() != null ? video.getFileName() : video.getName(); + if (fileName != null && !fileName.isEmpty()) { + currentPath.append("/").append(fileName); + KnowledgeSystemNode fileNode = findOrCreateChild(currentNode, fileName, + currentPath.toString(), KnowledgeSystemNode.NodeType.VIDEO_FILE); + fileNode.setVideoId(video.getId()); + fileNode.setVideo(video); + } + } + } + + return root; + + } catch (Exception e) { + logger.error("Error building knowledge path tree: " + e.getMessage(), e); + return null; + } + } + + /** + * Extracts ksKnowledgePath from document metadata + * @param doc The document to extract the path from + * @return The knowledge path or null if not present + */ + private String extractKnowledgePathFromMetadata(KSDocument doc) { + if (doc.getIngestionInfo() == null) { + return null; + } + + Map metadata = doc.getIngestionInfo().getMetadata(); + if (metadata == null) { + return null; + } + + return metadata.get("ksKnowledgePath"); + } + + /** + * Extracts ksKnowledgePath from video metadata + * @param video The video to extract the path from + * @return The knowledge path or null if not present + */ + private String extractKnowledgePathFromVideoMetadata(KSVideo video) { + if (video.getIngestionInfo() == null) { + return null; + } + + HashMap metadata = video.getIngestionInfo().getMetadata(); + if (metadata == null) { + return null; + } + + return metadata.get("ksKnowledgePath"); + } + + /** + * Helper method to find an existing child node or create a new one + */ + private KnowledgeSystemNode findOrCreateChild(KnowledgeSystemNode parent, String name, + String path, KnowledgeSystemNode.NodeType type) { + // Check if child already exists + for (KnowledgeSystemNode child : parent.getChildren()) { + if (child.getName().equals(name)) { + return child; + } + } + + // Create new child if not found + KnowledgeSystemNode newChild = new KnowledgeSystemNode(name, path, type); + parent.addChild(newChild); + return newChild; + } + + /** + * Deletes a document from the database and filesystem by its ID + * This method will be extended to handle cancellation from the search index + * @param documentId The ID of the document to delete + * @return true if the document was deleted successfully, false if the document was not found + */ + public boolean deleteDocument(String documentId) { + logger.info("Deleting document with ID: " + documentId); + + try { + // Check if document exists and retrieve it + KSDocument document = ksdocRepo.findById(documentId).orElse(null); + if (document == null) { + logger.warn("Document not found with ID: " + documentId); + return false; + } + + // Delete the physical file from filesystem + if (document.getFilePath() != null && !document.getFilePath().isEmpty()) { + try { + Path filePath = Paths.get(document.getFilePath()).normalize(); + if (Files.exists(filePath)) { + Files.delete(filePath); + logger.info("Physical file deleted successfully: " + document.getFilePath()); + } else { + logger.warn("Physical file not found at path: " + document.getFilePath()); + } + } catch (Exception fileException) { + logger.error("Error deleting physical file: " + document.getFilePath() + " - " + fileException.getMessage(), fileException); + // Continue with database deletion even if file deletion fails + } + } + + // Delete the document from the database + ksdocRepo.deleteById(documentId); + logger.info("Document deleted successfully from database with ID: " + documentId); + + // Delete from search index using the document ID + try { + // Extract project name from metadata + String projectName = "default"; + if (document.getIngestionInfo() != null && document.getIngestionInfo().getMetadata() != null) { + String metadataProjectName = document.getIngestionInfo().getMetadata().get("KsProjectName"); + if (metadataProjectName != null && !metadataProjectName.isEmpty()) { + projectName = metadataProjectName; + } + } + + ResponseEntity response = newDocumentServiceClient.deleteDocumentByKsDocumentId( + documentId, + projectName, + null // Use default Elasticsearch URL + ); + + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { + DeleteDocumentResponse deleteResponse = response.getBody(); + logger.info("Document deleted from search index. Deleted count: " + deleteResponse.getDeletedCount()); + } else { + logger.warn("Search index deletion returned non-successful status: " + response.getStatusCode()); + } + } catch (Exception indexException) { + logger.error("Error deleting document from search index: " + indexException.getMessage(), indexException); + // Continue even if search index deletion fails - document is already deleted from DB + } + + return true; + + } catch (Exception e) { + logger.error("Error deleting document with ID " + documentId + ": " + e.getMessage(), e); + return false; + } + } + + /** + * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections + * for a specific project + * + * @param projectName The name of the project to query metadata for + * @return Map where keys are metadata field names and values are sets of all possible values for each field + */ + public Map> getAllMetadataValues(String projectName) { + logger.info("Getting all metadata values for project: " + projectName); + + try { + // Map to store metadata keys and their distinct values + Map> metadataMap = new LinkedHashMap<>(); + + // Get metadata from documents + List documents = ksdocRepo.findAllMetadataByProjectName(projectName); + logger.info("Found " + documents.size() + " documents for project: " + projectName); + + for (KSDocument doc : documents) { + if (doc.getIngestionInfo() != null && doc.getIngestionInfo().getMetadata() != null) { + HashMap metadata = doc.getIngestionInfo().getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + + if (key != null && value != null && !value.trim().isEmpty()) { + metadataMap.computeIfAbsent(key, k -> new HashSet<>()).add(value); + } + } + } + } + + // Get metadata from videos + List videos = ksVideoRepo.findAllMetadataByProjectName(projectName); + logger.info("Found " + videos.size() + " videos for project: " + projectName); + + for (KSVideo video : videos) { + if (video.getIngestionInfo() != null && video.getIngestionInfo().getMetadata() != null) { + HashMap metadata = video.getIngestionInfo().getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + + if (key != null && value != null && !value.trim().isEmpty()) { + metadataMap.computeIfAbsent(key, k -> new HashSet<>()).add(value); + } + } + } + } + + logger.info("Total unique metadata keys found: " + metadataMap.size()); + + // Sort the values within each set for better presentation + Map> sortedMetadataMap = metadataMap.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().stream() + .sorted() + .collect(Collectors.toCollection(LinkedHashSet::new)), + (e1, e2) -> e1, + LinkedHashMap::new + )); + + return sortedMetadataMap; + + } catch (Exception e) { + logger.error("Error getting all metadata values for project " + projectName + ": " + e.getMessage(), e); + return new HashMap<>(); + } + } } diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 2a08941..3ae44c6 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -13,9 +13,6 @@ import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; import org.springframework.ai.transformer.splitter.TokenTextSplitter; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.ai.vectorstore.SearchRequest.Builder; -import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; @@ -47,9 +44,6 @@ public class KSIngestor { @Autowired private FileSystemStorageService storageService; - @Autowired - private VectorStore vectorStore; - @Value("${ksingestor.embedded.doc.batch.size:20}") private int embDocsBatchSize; @@ -60,14 +54,8 @@ public class KSIngestor { Logger logger = LoggerFactory.getLogger(KSIngestor.class); public void deleteAll(String document_file_name) { - Builder request_builder = SearchRequest.builder() - .query("*") - .similarityThreshold(0.0) - .filterExpression("'source'=='" + document_file_name + "'"); - SearchRequest request = request_builder.build(); - List docToDelete = vectorStore.similaritySearch(request); - - logger.info("Number of documents to delete: " + docToDelete.size()); + // VectorStore operations removed + logger.info("Document deletion requested for: " + document_file_name); } public IngestionOutput ingestLoop() { @@ -202,9 +190,9 @@ public class KSIngestor { splitDoc.getMetadata().putAll(metadata); } - ksDocument.setIngestionMessage("Embedding documents"); + ksDocument.setIngestionMessage("Processing documents"); ksDocumentRepository.save(ksDocument); - embedDocuments(splitDocs, ingestionInfo); + // VectorStore operations removed }); ksDocument.setIngestionStatus("INGESTED"); @@ -274,7 +262,7 @@ public class KSIngestor { splitDoc.getMetadata().putAll(meta2); docIndex++; } - embedtexts(splitDocs); + // VectorStore operations removed }); //ksTexts.setIngestionStatus("INGESTED"); @@ -293,55 +281,28 @@ public class KSIngestor { } private void embedtexts(List docs) { - - logger.info("Embedding texts"); - + logger.info("Processing texts"); docs.forEach(doc -> logger.info("text metadata: " + doc.getMetadata())); - try { - vectorStore.add(docs); - logger.info("Texts embedded"); - } catch (Exception e) { - logger.error("Error embedding Texts: ", e); - } + // VectorStore operations removed + logger.info("Texts processed"); } private void embedDocuments(List docs, KSIngestionInfo ingestionInfo) { - - logger.info("Embedding documents"); + logger.info("Processing documents"); int batchSize = embDocsBatchSize; for (int i = 0; i < docs.size(); i += batchSize) { int end = Math.min(i + batchSize, docs.size()); - List currentList = docs.subList(i, end); - try { - Thread.sleep(embDocRetryTime); - vectorStore.add(currentList); - logger.info("Documents embedded - Progress: Batch from {} to {} completed of {} total chunks", i, end, docs.size()); - } catch (Exception e) { - logger.error("Error embedding documents from {} to {}: {}", i, end, e.getMessage()); - } + logger.info("Documents processed - Progress: Batch from {} to {} completed of {} total chunks", i, end, docs.size()); } - + // VectorStore operations removed } public List testSimilaritySearch(String query,String filterQuery) { - Builder request_builder = SearchRequest.builder() - .query(query) - .topK(5) - .similarityThreshold(0.1); - - if(filterQuery != null && !filterQuery.isEmpty()){ - request_builder.filterExpression(filterQuery); - logger.info("Using Filter expression: " + filterQuery); - } - - SearchRequest request = request_builder.build(); - List docs = vectorStore.similaritySearch(request); - - logger.info("Number of VDB retrieved documents: " + docs.size()); - - return docs; + // VectorStore operations removed + logger.info("Similarity search requested for query: " + query); + return Collections.emptyList(); } public IngestionOutput setVideoInQueueIngestion(String id) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b286536..ea57d0b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -25,14 +25,6 @@ spring: api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - vectorstore: - chroma: - client: - host: "http://108.142.36.18" - port: "80" - key-token: "BxZWXFXC4UMSxamf5xP5SioGIg3FPfP7" - initialize-schema: "true" - collection-name: "olympus" data: mongodb: uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000 @@ -70,3 +62,13 @@ java-re-module: url: "http://localhost:8084" tika.config: "tika-config.xml" +new-document-service: + url: "http://localhost:8001" + +document: + storage: + path: /Users/andrea.terzani/Desktop/DEV/olympus/documents/apollo_storage/documents #C:\\repos\\olympus_ai\\documentStorage + +video: + storage: + path: /Users/andrea.terzani/Desktop/DEV/olympus/documents/apollo_storage/videos #C:\\repos\\olympus_ai\\videoStorage \ No newline at end of file diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..99b7fa7 --- /dev/null +++ b/start.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# ChromaDB Startup Script +# This script starts a ChromaDB instance using Docker + +echo "Starting ChromaDB instance..." + +# Check if Docker is running +if ! docker info > /dev/null 2>&1; then + echo "Error: Docker is not running. Please start Docker first." + exit 1 +fi + +# ChromaDB configuration +CHROMA_PORT=8000 +CHROMA_CONTAINER_NAME="chromadb-olympus" +CHROMA_IMAGE="ghcr.io/chroma-core/chroma:1.3.5" +CHROMA_DATA_DIR="./chroma-data" + +# Create data directory if it doesn't exist +mkdir -p "$CHROMA_DATA_DIR" + +# Check if container already exists +if [ "$(docker ps -aq -f name=$CHROMA_CONTAINER_NAME)" ]; then + echo "ChromaDB container already exists." + + # Check if it's running + if [ "$(docker ps -q -f name=$CHROMA_CONTAINER_NAME)" ]; then + echo "ChromaDB is already running on port $CHROMA_PORT" + echo "Access it at: http://localhost:$CHROMA_PORT" + exit 0 + else + echo "Starting existing ChromaDB container..." + docker start $CHROMA_CONTAINER_NAME + fi +else + echo "Creating and starting new ChromaDB container..." + docker run -d \ + --name $CHROMA_CONTAINER_NAME \ + -p $CHROMA_PORT:8000 \ + -v "$(pwd)/$CHROMA_DATA_DIR:/chroma/chroma" \ + $CHROMA_IMAGE +fi + +# Wait for ChromaDB to be ready +echo "Waiting for ChromaDB to be ready..." +sleep 3 + +# Check if ChromaDB is responding (with timeout) +if curl -s --max-time 5 http://localhost:$CHROMA_PORT/api/v1/heartbeat > /dev/null 2>&1; then + echo "✅ ChromaDB is up and running!" + echo " - URL: http://localhost:$CHROMA_PORT" + echo " - Data directory: $CHROMA_DATA_DIR" + echo " - Container name: $CHROMA_CONTAINER_NAME" + echo "" + echo "To stop ChromaDB, run: docker stop $CHROMA_CONTAINER_NAME" + echo "To view logs, run: docker logs -f $CHROMA_CONTAINER_NAME" +else + echo "⚠️ ChromaDB container started. Checking if it's ready..." + # Try without API endpoint - just check if container is running + if docker ps | grep -q $CHROMA_CONTAINER_NAME; then + echo "✅ ChromaDB container is running!" + echo " - URL: http://localhost:$CHROMA_PORT" + echo " - Data directory: $CHROMA_DATA_DIR" + echo " - Container name: $CHROMA_CONTAINER_NAME" + echo " - Note: API may take a few more seconds to be ready" + echo "" + echo "To stop ChromaDB, run: docker stop $CHROMA_CONTAINER_NAME" + echo "To view logs, run: docker logs -f $CHROMA_CONTAINER_NAME" + else + echo "❌ ChromaDB container failed to start" + echo " Check logs with: docker logs $CHROMA_CONTAINER_NAME" + exit 1 + fi +fi + +echo "" +echo "Update your application.yml with:" +echo " spring:" +echo " ai:" +echo " vectorstore:" +echo " chroma:" +echo " client:" +echo " host: \"http://localhost\"" +echo " port: \"$CHROMA_PORT\""