feat: Implement New Document Service Client for document deletion and agent search

feat: Add OpenAPI configuration for API documentation with security scheme

feat: Create KnowledgeTreeController for file and video uploads with enhanced metadata handling

feat: Define DTOs for agent search requests and responses, including metadata filtering

feat: Implement DeleteDocumentResponse DTO for document deletion responses

feat: Create KnowledgeSystemNode DTO for representing knowledge tree structure

chore: Add startup script for ChromaDB instance with Docker integration

apollo.code-workspace

@@ -0,0 +1,10 @@
+{
+	"folders": [
+		{
+			"path": "."
+		},
+		{
+			"path": "../olympus-common"
+		}
+	]
+}

pom.xml

@@ -88,11 +88,6 @@
 			<artifactId>spring-ai-starter-model-azure-openai</artifactId>
 		</dependency>
 
-		<dependency>
-			<groupId>org.springframework.ai</groupId>
-			<artifactId>spring-ai-starter-vector-store-chroma</artifactId>
-		</dependency>
-
 		<dependency>
 			<groupId>org.springframework.ai</groupId>
 			<artifactId>spring-ai-tika-document-reader</artifactId>

src/main/java/com/olympus/apollo/client/NewDocumentServiceClient.java

@@ -0,0 +1,146 @@
+package com.olympus.apollo.client;
+
+import com.olympus.apollo.dto.AgentSearchRequest;
+import com.olympus.apollo.dto.AgentSearchResponse;
+import com.olympus.apollo.dto.DeleteDocumentResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+@Service
+public class NewDocumentServiceClient {
+
+    private static final Logger logger = LoggerFactory.getLogger(NewDocumentServiceClient.class);
+
+    @Value("${new-document-service.url}")
+    private String serviceUrl;
+
+    private final RestTemplate restTemplate;
+
+    public NewDocumentServiceClient() {
+        SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
+        factory.setConnectTimeout(5000); // 5 seconds connection timeout
+        factory.setReadTimeout(60000); // 60 seconds read timeout
+        factory.setBufferRequestBody(false); // Don't buffer request body
+        
+        this.restTemplate = new RestTemplate(factory);
+        logger.info("RestTemplate initialized with timeouts - connect: 5s, read: 60s");
+    }
+
+    /**
+     * Delete all documents with the given KsDocumentId.
+     *
+     * This endpoint will:
+     * 1. Search for all documents with the specified KsDocumentId
+     * 2. Delete all matching documents from the index
+     * 3. Return the count of deleted documents
+     *
+     * @param ksDocumentId The KsDocumentId to delete
+     * @param project Project name for index isolation (default: "default")
+     * @param esUrl Elasticsearch URL (default: "http://localhost:9200")
+     * @return DeleteDocumentResponse with deletion details
+     */
+    public ResponseEntity<DeleteDocumentResponse> deleteDocumentByKsDocumentId(
+            String ksDocumentId, String project, String esUrl) {
+        
+        try {
+            // Build the URL with query parameters
+            UriComponentsBuilder uriBuilder = UriComponentsBuilder
+                    .fromHttpUrl(serviceUrl)
+                    .path("/api/documents/by-ks-document-id/{ks_document_id}")
+                    .queryParam("project", project != null ? project : "default");
+            
+            if (esUrl != null && !esUrl.isEmpty()) {
+                uriBuilder.queryParam("es_url", esUrl);
+            }
+            
+            String url = uriBuilder.buildAndExpand(ksDocumentId).toUriString();
+            
+            logger.info("Deleting document with KsDocumentId: {} from search index at URL: {}", ksDocumentId, url);
+            
+            // Execute DELETE request
+            ResponseEntity<DeleteDocumentResponse> response = restTemplate.exchange(
+                    url,
+                    HttpMethod.DELETE,
+                    null,
+                    DeleteDocumentResponse.class
+            );
+            
+            logger.info("Delete request completed with status: {}", response.getStatusCode());
+            return response;
+            
+        } catch (Exception e) {
+            logger.error("Error deleting document with KsDocumentId {}: {}", ksDocumentId, e.getMessage(), e);
+            throw e;
+        }
+    }
+
+    /**
+     * Unified search endpoint for AI agents.
+     * 
+     * Supports multiple search types:
+     * - semantic: Neural/vector similarity search
+     * - keyword: Exact term matching (AND operator)
+     * - fulltext: Analyzed text matching with fuzziness
+     * - hybrid: Combines semantic similarity with entity matching
+     * 
+     * @param searchRequest The search request containing query, project, and search parameters
+     * @param esUrl Elasticsearch URL (default: "http://localhost:9200")
+     * @return AgentSearchResponse with search results and formatted context
+     */
+    public ResponseEntity<AgentSearchResponse> agentSearch(
+            AgentSearchRequest searchRequest, String esUrl) {
+        
+        try {
+            // Build the URL with query parameters
+            UriComponentsBuilder uriBuilder = UriComponentsBuilder
+                    .fromHttpUrl(serviceUrl)
+                    .path("/api/agent/search");
+            
+           
+            String url = uriBuilder.toUriString();
+            
+            logger.info("Performing agent search for project: {} with query: {} using search type: {}", 
+                    searchRequest.getProject(), searchRequest.getQuery(), searchRequest.getSearchType());
+            
+            // Set up headers
+            HttpHeaders headers = new HttpHeaders();
+            headers.setContentType(MediaType.APPLICATION_JSON);
+            
+            // Create request entity
+            HttpEntity<AgentSearchRequest> requestEntity = new HttpEntity<>(searchRequest, headers);
+            
+            // Execute POST request
+            logger.info("Executing POST request to: {}", url);
+            ResponseEntity<AgentSearchResponse> response = restTemplate.exchange(
+                    url,
+                    HttpMethod.POST,
+                    requestEntity,
+                    AgentSearchResponse.class
+            );
+            
+            logger.info("Agent search HTTP call completed with status: {}", response.getStatusCode());
+            
+            if (response.getBody() != null) {
+                logger.info("Response body received with {} results", response.getBody().getTotalResults());
+            }
+            
+            logger.info("Returning response from client");
+            return response;
+            
+        } catch (Exception e) {
+            logger.error("Error performing agent search for project {}: {}", 
+                    searchRequest.getProject(), e.getMessage(), e);
+            throw e;
+        }
+    }
+}

src/main/java/com/olympus/apollo/config/OpenApiConfig.java

@@ -0,0 +1,32 @@
+package com.olympus.apollo.config;
+
+import io.swagger.v3.oas.models.Components;
+import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.info.Info;
+import io.swagger.v3.oas.models.security.SecurityRequirement;
+import io.swagger.v3.oas.models.security.SecurityScheme;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class OpenApiConfig {
+
+    @Bean
+    public OpenAPI customOpenAPI() {
+        final String securitySchemeName = "bearerAuth";
+        
+        return new OpenAPI()
+                .info(new Info()
+                        .title("Apollo API")
+                        .version("1.0")
+                        .description("Apollo Service API Documentation"))
+                .addSecurityItem(new SecurityRequirement().addList(securitySchemeName))
+                .components(new Components()
+                        .addSecuritySchemes(securitySchemeName, new SecurityScheme()
+                                .name(securitySchemeName)
+                                .type(SecurityScheme.Type.HTTP)
+                                .scheme("bearer")
+                                .bearerFormat("JWT")
+                                .description("Enter JWT token obtained from /api/auth/login endpoint")));
+    }
+}

src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java

@@ -3,10 +3,13 @@ package com.olympus.apollo.controllers.FeApi;
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.web.bind.annotation.*;
 
+import com.olympus.apollo.dto.KnowledgeSystemNode;
 import com.olympus.model.apollo.KSDocument;
 import com.olympus.apollo.repository.KSDocumentRepository;
+import com.olympus.apollo.security.entity.User;
 import com.olympus.apollo.services.KSDocumentService;
 import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
@@ -45,6 +48,23 @@ public class KsDocumentController {
 
         return ksDocumentService.downloadKSDocument(doc);    
 
-}
+    }
+
+    /**
+     * Deletes a document by its ID
+     * The service method will be extended to handle search index cancellation
+     * @param id The ID of the document to delete
+     * @return ResponseEntity indicating success or failure
+     */
+    @DeleteMapping("/{id}")
+    public ResponseEntity<Void> deleteDocument(@PathVariable String id) {
+        boolean deleted = ksDocumentService.deleteDocument(id);
+        
+        if (deleted) {
+            return ResponseEntity.ok().build();
+        } else {
+            return ResponseEntity.notFound().build();
+        }
+    }
 
 }

src/main/java/com/olympus/apollo/controllers/KSFileController.java

@@ -15,6 +15,7 @@ import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.ExceptionHandler;
@@ -52,13 +53,15 @@ public class KSFileController {
 
     private static final Logger logger = LoggerFactory.getLogger(KSFileController.class);
 
-    private String document_path="/mnt/apollo_storage/documents";
+    @Value("${document.storage.path}")
+    private String document_path;
 
 
     @PostMapping("/upload")
     public ResponseEntity<KSDocument> handleFileUpload(
             @RequestParam("file") MultipartFile file,
-            @ModelAttribute FileUploadDTO fileUploadDTO
+            @ModelAttribute FileUploadDTO fileUploadDTO,
+            @RequestParam(value = "version", required = false) String version
     ) {
 
         try (InputStream inputStream = file.getInputStream()) {
@@ -88,7 +91,13 @@ public class KSFileController {
             ksDocument.setFileName(file.getOriginalFilename());
             ksDocument.setName(file.getOriginalFilename());
             ksDocument.setDescription(fileUploadDTO.getDescription());
-            ksDocument.setIngestionStatus("INGESTION_QUEUE");
+            
+            if (version != null && version.contains("v2")) {
+                ksDocument.setIngestionStatus("IGNORE_INGESTION");
+                ksDocument.setIngestionStatusV2("INGESTION_QUEUE");
+            } else {
+                ksDocument.setIngestionStatus("INGESTION_QUEUE");
+            }
             ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
 
             Date now = new Date();
@@ -104,6 +113,14 @@ public class KSFileController {
             metadata.put("KsFileSource", file.getOriginalFilename());
             metadata.put("KsProjectName", fileUploadDTO.getKsProjectName());
             
+            logger.info("[UPLOAD] ksKnowledgePath received: {}", fileUploadDTO.getKsKnowledgePath());
+            if (fileUploadDTO.getKsKnowledgePath() != null && !fileUploadDTO.getKsKnowledgePath().isEmpty()) {
+                metadata.put("ksKnowledgePath", fileUploadDTO.getKsKnowledgePath());
+                logger.info("[UPLOAD] Added ksKnowledgePath to metadata: {}", fileUploadDTO.getKsKnowledgePath());
+            } else {
+                logger.warn("[UPLOAD] ksKnowledgePath is null or empty");
+            }
+
             ksIngestionInfo.setMetadata(metadata);
             ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize());
             ksIngestionInfo.setMinChunkSize(fileUploadDTO.getMinChunkSize());

src/main/java/com/olympus/apollo/controllers/KnowledgeTreeController.java

@@ -0,0 +1,393 @@
+package com.olympus.apollo.controllers;
+
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.ModelAttribute;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.olympus.apollo.client.NewDocumentServiceClient;
+import com.olympus.apollo.dto.AgentSearchRequest;
+import com.olympus.apollo.dto.AgentSearchResponse;
+import com.olympus.apollo.dto.KnowledgeSystemNode;
+import com.olympus.apollo.exception.StorageException;
+import com.olympus.apollo.repository.KSDocumentRepository;
+import com.olympus.apollo.repository.KSVideoRepository;
+import com.olympus.apollo.security.entity.User;
+import com.olympus.apollo.services.KSDocumentService;
+import com.olympus.dto.FileUploadDTO;
+import com.olympus.dto.VideoUploadDTO;
+import com.olympus.model.apollo.KSDocument;
+import com.olympus.model.apollo.KSIngestionInfo;
+import com.olympus.model.apollo.KSVideo;
+import com.olympus.model.apollo.KSVideoIngestionInfo;
+
+/**
+ * KnowledgeTree Controller - V2 API
+ * This controller manages the new version of Knowledge Tree operations including
+ * file uploads and video uploads for the new application flow.
+ * Legacy endpoints remain in KSFileController and VideoController for backward compatibility.
+ */
+@RestController
+@RequestMapping("/api/v2/knowledgetree")
+public class KnowledgeTreeController {
+
+    @Autowired
+    private KSDocumentRepository ksDocumentRepository;
+
+    @Autowired
+    private KSVideoRepository ksVideoRepository;
+
+    @Autowired
+    private KSDocumentService ksDocumentService;
+
+    @Autowired
+    private NewDocumentServiceClient newDocumentServiceClient;
+
+    @Value("${document.storage.path}")
+    private String documentStoragePath;
+
+    @Value("${video.storage.path}")
+    private String videoStoragePath;
+
+    private static final Logger logger = LoggerFactory.getLogger(KnowledgeTreeController.class);
+
+    /**
+     * Upload a file to the knowledge tree (V2)
+     * Enhanced version with improved metadata handling and knowledge path support
+     * 
+     * @param file The file to upload
+     * @param fileUploadDTO Upload metadata
+     * @return ResponseEntity containing the created KSDocument
+     */
+    @PostMapping("/files/upload")
+    public ResponseEntity<KSDocument> handleFileUpload(
+            @RequestParam("file") MultipartFile file,
+            @ModelAttribute FileUploadDTO fileUploadDTO) {
+
+        logger.info("[V2-UPLOAD] Starting file upload for: {}", file.getOriginalFilename());
+        logger.info("[V2-UPLOAD] Project: {}, Knowledge Path: {}", 
+                    fileUploadDTO.getKsProjectName(), 
+                    fileUploadDTO.getKsKnowledgePath());
+
+        try (InputStream inputStream = file.getInputStream()) {
+            // Validate file
+            if (file.isEmpty()) {
+                logger.error("[V2-UPLOAD] File is empty: {}", file.getOriginalFilename());
+                throw new StorageException("Failed to store empty file.");
+            }
+
+            // Create storage directory
+            Path folderPath = Paths.get(documentStoragePath)
+                    .resolve(fileUploadDTO.getKsProjectName())
+                    .normalize()
+                    .toAbsolutePath();
+
+            Files.createDirectories(folderPath);
+            logger.info("[V2-UPLOAD] Created folder: {}", folderPath);
+
+            // Store file
+            String filePath = folderPath.resolve(file.getOriginalFilename()).toString();
+            try (OutputStream outputStream = new FileOutputStream(filePath)) {
+                byte[] buffer = new byte[8 * 1024];
+                int bytesRead;
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    outputStream.write(buffer, 0, bytesRead);
+                }
+                logger.info("[V2-UPLOAD] File stored successfully at: {}", filePath);
+            }
+
+            // Create KSDocument entity with V2 ingestion status
+            KSDocument ksDocument = new KSDocument();
+            ksDocument.setFilePath(filePath);
+            ksDocument.setFileName(file.getOriginalFilename());
+            ksDocument.setName(file.getOriginalFilename());
+            ksDocument.setDescription(fileUploadDTO.getDescription());
+            
+            // V2 uses new ingestion flow
+            ksDocument.setIngestionStatus("IGNORE_INGESTION");
+            ksDocument.setIngestionStatusV2("INGESTION_QUEUE");
+            
+            Date now = new Date();
+            ksDocument.setIngestionDate(now);
+            ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(now));
+
+            // Build ingestion info with enhanced metadata
+            KSIngestionInfo ksIngestionInfo = new KSIngestionInfo();
+            ksIngestionInfo.setType(fileUploadDTO.getType());
+
+            HashMap<String, String> metadata = new HashMap<>();
+            metadata.put("KsApplicationName", fileUploadDTO.getKsApplicationName());
+            metadata.put("KsDoctype", fileUploadDTO.getKsDocType());
+            metadata.put("KsDocSource", fileUploadDTO.getKsDocSource());
+            metadata.put("KsFileSource", file.getOriginalFilename());
+            metadata.put("KsProjectName", fileUploadDTO.getKsProjectName());
+            
+            // Knowledge path support for V2
+            if (fileUploadDTO.getKsKnowledgePath() != null && !fileUploadDTO.getKsKnowledgePath().isEmpty()) {
+                metadata.put("ksKnowledgePath", fileUploadDTO.getKsKnowledgePath());
+                logger.info("[V2-UPLOAD] Knowledge path added: {}", fileUploadDTO.getKsKnowledgePath());
+            } else {
+                logger.warn("[V2-UPLOAD] No knowledge path provided");
+            }
+
+            ksIngestionInfo.setMetadata(metadata);
+            ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize());
+            ksIngestionInfo.setMinChunkSize(fileUploadDTO.getMinChunkSize());
+            ksIngestionInfo.setMaxNumberOfChunks(fileUploadDTO.getMaxNumberOfChunks());
+            ksIngestionInfo.setMinChunkSizeToEmbed(fileUploadDTO.getMinChunkSizeToEmbed());
+
+            ksDocument.setIngestionInfo(ksIngestionInfo);
+            ksDocumentRepository.save(ksDocument);
+
+            logger.info("[V2-UPLOAD] Document saved successfully with ID: {}", ksDocument.getId());
+            return ResponseEntity.ok(ksDocument);
+            
+        } catch (StorageException e) {
+            logger.error("[V2-UPLOAD] Storage exception: {}", e.getMessage(), e);
+            return ResponseEntity.status(400).body(null);
+        } catch (Exception e) {
+            logger.error("[V2-UPLOAD] Unexpected error during file upload: {}", e.getMessage(), e);
+            return ResponseEntity.status(500).body(null);
+        }
+    }
+
+    /**
+     * Upload a video to the knowledge tree (V2)
+     * Enhanced version with improved metadata handling
+     * 
+     * @param file The video file to upload
+     * @param videoUploadDTO Upload metadata
+     * @return ResponseEntity containing the created KSVideo
+     */
+    @PostMapping("/videos/upload")
+    public ResponseEntity<KSVideo> handleVideoUpload(
+            @RequestParam("file") MultipartFile file,
+            @ModelAttribute VideoUploadDTO videoUploadDTO) {
+
+        logger.info("[V2-VIDEO-UPLOAD] Starting video upload for: {}", file.getOriginalFilename());
+        logger.info("[V2-VIDEO-UPLOAD] Project: {}, Video Group: {}", 
+                    videoUploadDTO.getKsProjectName(), 
+                    videoUploadDTO.getKsVideoGroupId());
+
+        try (InputStream inputStream = file.getInputStream()) {
+            // Validate file
+            if (file.isEmpty()) {
+                logger.error("[V2-VIDEO-UPLOAD] File is empty: {}", file.getOriginalFilename());
+                throw new StorageException("Failed to store empty file.");
+            }
+
+            // Create storage directory
+            Path folderPath = Paths.get(videoStoragePath)
+                    .resolve(videoUploadDTO.getKsProjectName())
+                    .normalize()
+                    .toAbsolutePath();
+
+            Files.createDirectories(folderPath);
+            logger.info("[V2-VIDEO-UPLOAD] Created folder: {}", folderPath);
+
+            // Store video file
+            String filePath = folderPath.resolve(file.getOriginalFilename()).toString();
+            try (OutputStream outputStream = new FileOutputStream(filePath)) {
+                byte[] buffer = new byte[8 * 1024];
+                int bytesRead;
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    outputStream.write(buffer, 0, bytesRead);
+                }
+                logger.info("[V2-VIDEO-UPLOAD] Video stored successfully at: {}", filePath);
+            }
+
+            // Create KSVideo entity
+            KSVideo ksVideo = new KSVideo();
+            ksVideo.setFilePath(filePath);
+            ksVideo.setFileName(file.getOriginalFilename());
+            ksVideo.setName(file.getOriginalFilename());
+            ksVideo.setDescription(videoUploadDTO.getDescription());
+            ksVideo.setIngestionStatus("IGNORE_INGESTION");
+            ksVideo.setIngestionStatusV2("INGESTION_QUEUE");
+            Date now = new Date();
+            ksVideo.setIngestionDate(now);
+            ksVideo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(now));
+
+            // Build video ingestion info
+            KSVideoIngestionInfo ksVideoIngestionInfo = new KSVideoIngestionInfo();
+            ksVideoIngestionInfo.setType(videoUploadDTO.getType());
+
+            HashMap<String, String> metadata = new HashMap<>();
+            metadata.put("KsApplicationName", videoUploadDTO.getKsApplicationName());
+            metadata.put("KsDoctype", videoUploadDTO.getKsDocType());
+            metadata.put("KsDocSource", videoUploadDTO.getKsDocSource());
+            metadata.put("KsFileSource", file.getOriginalFilename());
+            metadata.put("KsVideoGroupId", videoUploadDTO.getKsVideoGroupId());
+            metadata.put("KsProjectName", videoUploadDTO.getKsProjectName());
+            
+            // Knowledge path support for V2 videos
+            if (videoUploadDTO.getKsKnowledgePath() != null && !videoUploadDTO.getKsKnowledgePath().isEmpty()) {
+                metadata.put("ksKnowledgePath", videoUploadDTO.getKsKnowledgePath());
+                logger.info("[V2-VIDEO-UPLOAD] Knowledge path added: {}", videoUploadDTO.getKsKnowledgePath());
+            } else {
+                logger.warn("[V2-VIDEO-UPLOAD] No knowledge path provided");
+            }
+
+            ksVideoIngestionInfo.setMetadata(metadata);
+            ksVideoIngestionInfo.setNumberOfChunkToEmbed(videoUploadDTO.getNumberOfChunkToEmbed());
+            ksVideoIngestionInfo.setChunkDurationInSeconds(videoUploadDTO.getChunkDurationInSeconds());
+            ksVideoIngestionInfo.setLanguage(videoUploadDTO.getLanguage());
+
+            ksVideo.setIngestionInfo(ksVideoIngestionInfo);
+            ksVideoRepository.save(ksVideo);
+
+            logger.info("[V2-VIDEO-UPLOAD] Video saved successfully with ID: {}", ksVideo.getId());
+            return ResponseEntity.ok(ksVideo);
+
+        } catch (StorageException e) {
+            logger.error("[V2-VIDEO-UPLOAD] Storage exception: {}", e.getMessage(), e);
+            return ResponseEntity.status(400).body(null);
+        } catch (Exception e) {
+            logger.error("[V2-VIDEO-UPLOAD] Unexpected error during video upload: {}", e.getMessage(), e);
+            return ResponseEntity.status(500).body(null);
+        }
+    }
+
+    /**
+     * Returns a filesystem-like tree structure based on ksKnowledgePath metadata
+     * for the current user's selected project
+     * @return KnowledgeSystemNode representing the root of the tree
+     */
+    @GetMapping("/knowledge-tree")
+    public ResponseEntity<KnowledgeSystemNode> getKnowledgeTree() {
+        User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
+        String projectName = principal.getSelectedProject().getInternal_name();
+        
+        KnowledgeSystemNode tree = ksDocumentService.buildKnowledgePathTree(projectName);
+        
+        if (tree == null) {
+            return ResponseEntity.noContent().build();
+        }
+        
+        return ResponseEntity.ok(tree);
+    }
+
+    /**
+     * Returns a filesystem-like tree structure based on ksKnowledgePath metadata
+     * for a specific project
+     * @param projectName The name of the project
+     * @return KnowledgeSystemNode representing the root of the tree
+     */
+    @GetMapping("/knowledge-tree/{projectName}")
+    public ResponseEntity<KnowledgeSystemNode> getKnowledgeTreeByProject(@PathVariable String projectName) {
+        KnowledgeSystemNode tree = ksDocumentService.buildKnowledgePathTree(projectName);
+        
+        if (tree == null) {
+            return ResponseEntity.noContent().build();
+        }
+        
+        return ResponseEntity.ok(tree);
+    }
+
+
+
+    /**
+     * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections
+     * for the current user's selected project
+     * 
+     * @return Map where keys are metadata field names and values are sets of all possible values for each field
+     */
+    @GetMapping("/metadata/all-values")
+    public ResponseEntity<Map<String, Set<String>>> getAllMetadataValues() {
+        User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
+        String projectName = principal.getSelectedProject().getInternal_name();
+        
+        Map<String, Set<String>> allMetadata = ksDocumentService.getAllMetadataValues(projectName);
+        
+        if (allMetadata == null || allMetadata.isEmpty()) {
+            return ResponseEntity.noContent().build();
+        }
+        
+        return ResponseEntity.ok(allMetadata);
+    }
+
+    /**
+     * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections
+     * for a specific project
+     * 
+     * @param projectName The name of the project
+     * @return Map where keys are metadata field names and values are sets of all possible values for each field
+     */
+    @GetMapping("/metadata/all-values/{projectName}")
+    public ResponseEntity<Map<String, Set<String>>> getAllMetadataValuesByProject(@PathVariable String projectName) {
+        Map<String, Set<String>> allMetadata = ksDocumentService.getAllMetadataValues(projectName);
+        
+        if (allMetadata == null || allMetadata.isEmpty()) {
+            return ResponseEntity.noContent().build();
+        }
+        
+        return ResponseEntity.ok(allMetadata);
+    }
+
+    /**
+     * Agent search endpoint - Unified search for AI agents
+     * Supports multiple search types: semantic, keyword, fulltext, and hybrid
+     * 
+     * @param searchRequest The search request containing query, project, and search parameters
+     * @param esUrl Optional Elasticsearch URL (defaults to service configuration)
+     * @return ResponseEntity containing AgentSearchResponse with search results and formatted context
+     */
+    @PostMapping("/agent-search")
+    public ResponseEntity<AgentSearchResponse> agentSearch(
+            @RequestBody AgentSearchRequest searchRequest) {
+        
+        logger.info("[AGENT-SEARCH] Received search request for project: {}, query: {}, type: {}", 
+                    searchRequest.getProject(), 
+                    searchRequest.getQuery(), 
+                    searchRequest.getSearchType());
+
+        try {
+            logger.info("[AGENT-SEARCH] Calling document service client...");
+            ResponseEntity<AgentSearchResponse> response = 
+                    newDocumentServiceClient.agentSearch(searchRequest, null);
+            
+            logger.info("[AGENT-SEARCH] Client call completed with status: {}", response.getStatusCode());
+            
+            AgentSearchResponse body = response.getBody();
+            if (body != null) {
+                logger.info("[AGENT-SEARCH] Search completed successfully. Results: {}", 
+                           body.getTotalResults());
+            }
+            
+            logger.info("[AGENT-SEARCH] Creating new ResponseEntity with body");
+            // Create a new ResponseEntity to avoid serialization issues with the proxied response
+            return ResponseEntity
+                    .status(response.getStatusCode())
+                    .headers(response.getHeaders())
+                    .body(body);
+            
+        } catch (Exception e) {
+            logger.error("[AGENT-SEARCH] Error performing agent search: {}", e.getMessage(), e);
+            return ResponseEntity.status(500).body(null);
+        }
+    }
+
+}

src/main/java/com/olympus/apollo/controllers/SearchDocController.java

@@ -1,14 +1,9 @@
 package com.olympus.apollo.controllers;
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.slf4j.LoggerFactory;
-import org.springframework.ai.document.Document;
-import org.springframework.ai.vectorstore.SearchRequest;
-import org.springframework.ai.vectorstore.SearchRequest.Builder;
-import org.springframework.ai.vectorstore.VectorStore;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -22,34 +17,12 @@ import ch.qos.logback.classic.Logger;
 @RequestMapping("/api/vsearch")
 public class SearchDocController {
 
-    @Autowired
-    private VectorStore vectorStore;
-
     Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class);
     
     @PostMapping("/doc_search")
     public List<String> vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) {
-
+        // VectorStore operations removed
-        Builder request_builder = SearchRequest.builder()
+        logger.info("Vector search requested for query: " + vectorSearchRequest.getQuery());
-        .query(vectorSearchRequest.getQuery())
+        return Collections.emptyList();
-        .topK(vectorSearchRequest.getTopK())
-        .similarityThreshold(vectorSearchRequest.getThreshold());
-
-        if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){
-            request_builder.filterExpression(vectorSearchRequest.getFilterQuery());
-            logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery());
-        } 
-
-        SearchRequest request  =  request_builder.build();
-        List<Document> docs  = this.vectorStore.similaritySearch(request);
-        
-        logger.info("Number of VDB retrieved documents: " + docs.size());
-
-        List<String> results = new ArrayList<String>();
-        for (Document doc : docs) {
-            results.add(doc.getText());
-        }
-       return results;
-        
     }
 }

src/main/java/com/olympus/apollo/dto/AgentSearchRequest.java

@@ -0,0 +1,119 @@
+package com.olympus.apollo.dto;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Map;
+
+/**
+ * Request model for agent search endpoint.
+ * Supports multiple search types: semantic, keyword, fulltext, or hybrid.
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AgentSearchRequest {
+
+    /**
+     * Project name (required)
+     */
+    private String project;
+
+    /**
+     * Search query text
+     */
+    private String query;
+
+    /**
+     * Type of search: semantic, keyword, fulltext, or hybrid
+     * Default: hybrid
+     */
+    @JsonProperty("search_type")
+    @Builder.Default
+    private String searchType = "hybrid";
+
+    /**
+     * Number of results to return (1-50)
+     * Default: 5
+     */
+    @Builder.Default
+    private Integer k = 5;
+
+    /**
+     * Maximum characters for formatted context (100-50000)
+     * Default: 4000
+     */
+    @JsonProperty("max_context_chars")
+    @Builder.Default
+    private Integer maxContextChars = 4000;
+
+    /**
+     * Weight for vector similarity in hybrid search (0.0-1.0)
+     * Default: 0.7
+     */
+    @JsonProperty("vector_weight")
+    @Builder.Default
+    private Double vectorWeight = 0.7;
+
+    /**
+     * Weight for entity matching in hybrid search (0.0-1.0)
+     * Default: 0.3
+     */
+    @JsonProperty("entity_weight")
+    @Builder.Default
+    private Double entityWeight = 0.3;
+
+    /**
+     * Filter by application name
+     */
+    @JsonProperty("KsApplicationName")
+    private String ksApplicationName;
+
+    /**
+     * Filter by document type
+     */
+    @JsonProperty("KsDoctype")
+    private String ksDoctype;
+
+    /**
+     * Filter by document source
+     */
+    @JsonProperty("KsDocSource")
+    private String ksDocSource;
+
+    /**
+     * Filter by file source
+     */
+    @JsonProperty("KsFileSource")
+    private String ksFileSource;
+
+    /**
+     * Filter by document ID
+     */
+    @JsonProperty("KsDocumentId")
+    private String ksDocumentId;
+
+    /**
+     * Filter by project name in metadata
+     */
+    @JsonProperty("KsProjectName")
+    private String ksProjectName;
+
+    /**
+     * Filter by knowledge folder
+     */
+    @JsonProperty("KsKnowledgePath")
+    private String ksKnowledgePath;
+
+    /**
+     * Filter by configurable tags
+     * Example: {"scope": "finance", "requirement_id": ["ID12211"]}
+     */
+    private Map<String, Object> tags;
+}

src/main/java/com/olympus/apollo/dto/AgentSearchResponse.java

@@ -0,0 +1,142 @@
+package com.olympus.apollo.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Response model for agent search endpoint.
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class AgentSearchResponse {
+
+    /**
+     * The original search query
+     */
+    private String query;
+
+    /**
+     * The project searched
+     */
+    private String project;
+
+    /**
+     * The search type used
+     */
+    @JsonProperty("search_type")
+    private String searchType;
+
+    /**
+     * Total number of results found
+     */
+    @JsonProperty("total_results")
+    private Integer totalResults;
+
+    /**
+     * List of search results
+     */
+    private List<SearchResultItem> results;
+
+    /**
+     * Pre-formatted context for LLM consumption
+     */
+    @JsonProperty("formatted_context")
+    private String formattedContext;
+
+    /**
+     * Additional metadata about the search
+     */
+    private Map<String, Object> metadata;
+
+    /**
+     * Individual search result item
+     */
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static class SearchResultItem {
+        
+        /**
+         * Content of the document chunk
+         */
+        private String content;
+
+        /**
+         * Relevance score
+         */
+        private Double score;
+
+        /**
+         * File name
+         */
+        @JsonProperty("file_name")
+        private String fileName;
+
+        /**
+         * Chunk index within the document
+         */
+        @JsonProperty("chunk_index")
+        private Integer chunkIndex;
+
+        /**
+         * Entities found in this chunk
+         */
+        private List<Map<String, Object>> entities;
+
+        /**
+         * Tags associated with this result
+         */
+        private Map<String, Object> tags;
+
+        /**
+         * Knowledge path
+         */
+        @JsonProperty("ks_knowledge_path")
+        private String ksKnowledgePath;
+
+        /**
+         * Document type
+         */
+        @JsonProperty("ks_doctype")
+        private String ksDoctype;
+
+        /**
+         * Application name
+         */
+        @JsonProperty("ks_application_name")
+        private String ksApplicationName;
+
+        /**
+         * Document source
+         */
+        @JsonProperty("ks_doc_source")
+        private String ksDocSource;
+
+        /**
+         * File source
+         */
+        @JsonProperty("ks_file_source")
+        private String ksFileSource;
+
+        /**
+         * Document ID
+         */
+        @JsonProperty("ks_document_id")
+        private String ksDocumentId;
+
+        /**
+         * Project name
+         */
+        @JsonProperty("ks_project_name")
+        private String ksProjectName;
+    }
+}

src/main/java/com/olympus/apollo/dto/DeleteDocumentResponse.java

@@ -0,0 +1,13 @@
+package com.olympus.apollo.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class DeleteDocumentResponse {
+    private String message;
+    private int deletedCount;
+    private String ksDocumentId;
+    private String project;
+}

src/main/java/com/olympus/apollo/dto/KnowledgeSystemNode.java

@@ -0,0 +1,44 @@
+package com.olympus.apollo.dto;
+
+import com.olympus.model.apollo.KSDocument;
+import com.olympus.model.apollo.KSVideo;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Getter 
+@Setter
+public class KnowledgeSystemNode {
+    
+    private String name;
+    private String path;
+    private NodeType type;
+    private List<KnowledgeSystemNode> children;
+    private String documentId;
+    private KSDocument document;
+    private String videoId;
+    private KSVideo video;
+    
+    public KnowledgeSystemNode() {
+        this.children = new ArrayList<>();
+    }
+    
+    public KnowledgeSystemNode(String name, String path, NodeType type) {
+        this.name = name;
+        this.path = path;
+        this.type = type;
+        this.children = new ArrayList<>();
+    }
+    
+    public void addChild(KnowledgeSystemNode child) {
+        this.children.add(child);
+    }
+    
+    public enum NodeType {
+        FOLDER,
+        FILE,
+        VIDEO_FILE
+    }
+}

src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java

@@ -6,7 +6,6 @@ import java.util.List;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
-import org.springframework.data.repository.query.Param;
 import org.springframework.data.rest.core.annotation.RepositoryRestResource;
 import org.springframework.web.bind.annotation.CrossOrigin;
 
@@ -24,4 +23,10 @@ public interface KSDocumentRepository extends MongoRepository<KSDocument, String
     @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0 }")
     public List<KSDocument> findByProjectName(String projectName, Sort sort);
 
+    @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0 }")
+    public List<KSDocument> findAllByProjectName(String projectName);
+
+    @Query(value = "{ 'ingestionInfo.metadata.KsProjectName': ?0 }", fields = "{ 'ingestionInfo.metadata': 1 }")
+    public List<KSDocument> findAllMetadataByProjectName(String projectName);
+
 }

src/main/java/com/olympus/apollo/repository/KSVideoRepository.java

@@ -34,4 +34,7 @@ public interface KSVideoRepository extends MongoRepository<KSVideo, String> {
     })
     List<VideoGroupCount> countVideosByGroupIds(List<String> groupIds);
 
+    @Query(value = "{ 'ingestionInfo.metadata.KsProjectName': ?0 }", fields = "{ 'ingestionInfo.metadata': 1 }")
+    public List<KSVideo> findAllMetadataByProjectName(String projectName);
+
 }

src/main/java/com/olympus/apollo/services/DeletionService.java

@@ -1,21 +1,16 @@
 package com.olympus.apollo.services;
 
 import java.util.Date;
-import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.ai.document.Document;
-import org.springframework.ai.vectorstore.SearchRequest;
-import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.messaging.simp.SimpMessagingTemplate;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
-import com.olympus.apollo.exception.vectorStoreMetaDetailsEmptyException;
 import com.olympus.apollo.repository.KSDocumentRepository;
 import com.olympus.apollo.repository.KSGitInfoRepository;
 import com.olympus.apollo.repository.KSGitIngestionInfoRepository;
@@ -51,17 +46,10 @@ public class DeletionService {
     @Autowired
     private SimpMessagingTemplate simpMessagingTemplate;
 
-    @Autowired
-    private VectorStore vectorStore;
-
     @Async("asyncTaskExecutor")
     public void deleteRecords(DeletionRequest deletionRequest) {
         try {
-
-            String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'";
             logger.info("Starting deletion");
-            vectorStore.delete(rag_filter);
-
             ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId());
             logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId());
         } catch (Exception e) {
@@ -79,36 +67,13 @@ public class DeletionService {
 
             logger.info("Starting deletion");
             
-            String rag_filter = "KsDocumentId=='" + deletionRequest.getKsDocumentId() + "'";
+            // VectorStore operations removed
-
-
-            SearchRequest searchRequest = SearchRequest.builder()
-            .query(" ")
-            .filterExpression(rag_filter)
-            .topK(Integer.MAX_VALUE)
-            .build();
-
-            List<String> idsToDelete = vectorStore.similaritySearch(searchRequest)
-                                                .stream()
-                                                .map(Document::getId)
-                                                .toList();
-
-            logger.info("Found {} documents to delete for KsDocumentId: {}", idsToDelete.size(), deletionRequest.getKsDocumentId());
-
-            //Batch per eliminare i file con più richieste
-            final int DELETE_BATCH_SIZE = 500;
-            for (int i = 0; i < idsToDelete.size(); i += DELETE_BATCH_SIZE) {
-                int end = Math.min(i + DELETE_BATCH_SIZE, idsToDelete.size());
-                List<String> batch = idsToDelete.subList(i, end);
-                logger.info("Deleting batch from {} to {}", i, end);
-                vectorStore.delete(batch);
-            }
             
             ksDocument.setIngestionStatus("LOADED");
             ksDocument.setIngestionDate(new Date());
             ksDocumentRepository.save(ksDocument);
 
-            logger.info("KSDocument with id {} deleted from VectorStore successfully.", deletionRequest.getKsDocumentId());
+            logger.info("KSDocument with id {} status updated successfully.", deletionRequest.getKsDocumentId());
 
         } catch (Exception e) {
             logger.error("An error occurred while deleting records: ", e);
@@ -119,16 +84,9 @@ public class DeletionService {
     @Async("asyncTaskExecutor")
     public void deleteVideoRecords(DeletionRequest deletionRequest) {
         try {
-
-            String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'";
             logger.info("Starting deletion");
-            vectorStore.delete(rag_filter);
-
             ksVideoRepository.deleteById(deletionRequest.getKsDocumentId());
             logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId());
-            // }else{
-            //     logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId());
-            // }
         } catch (Exception e) {
             logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName());
             throw new RuntimeException("An error occurred while deleting records", e);
@@ -138,26 +96,20 @@ public class DeletionService {
     @Async("asyncTaskExecutor")
     public void deleteVideoRecordsOnlyFromVectorStore(DeletionRequest deletionRequest) {
         try {
-
             KSVideo ksVideo = ksVideoRepository.findById(deletionRequest.getKsDocumentId()).get();
             ksVideo.setIngestionStatus("DELETING");
             ksVideoRepository.save(ksVideo);
 
-            String rag_filter = "KsDocumentId=='"+deletionRequest.getKsDocumentId()+"'";
-
             logger.info("Starting deletion");
-            vectorStore.delete(rag_filter);
             
-            //elimino dal vectorStore ma mantengo il record
+            // VectorStore operations removed
+            
             ksVideo.setIngestionStatus("LOADED");
             Date now = new Date();
             ksVideo.setIngestionDate(now);
 
             ksVideoRepository.save(ksVideo);
-            logger.info("KSVideo with id {} deleted from VectorStore successfully.", deletionRequest.getKsDocumentId());
+            logger.info("KSVideo with id {} status updated successfully.", deletionRequest.getKsDocumentId());
-            // }else{
-            //     logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId());
-            // }
         } catch (Exception e) {
             logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName());
             throw new RuntimeException("An error occurred while deleting records", e);
@@ -179,7 +131,6 @@ public class DeletionService {
 
                 boolean KSGitInfoExists = ksGitInfoId != null && !ksGitInfoId.isEmpty() && ksGitInfoRepository.existsById(ksGitInfoId);
                 boolean KSGitIngestionInfoExists = ksGitIngestionInfoId != null && !ksGitIngestionInfoId.isEmpty() && ksGitIngestionInfoRepository.existsById(ksGitIngestionInfoId);
-                boolean vectorStoreGitDetailsExists = applicationName != null && ksDocSource != null && ksFileSource != null && ksDoctype != null && ksBranch != null;
 
                 logger.info("KSGitInfo with id {} exists: {}", ksGitInfoId,KSGitInfoExists);
                 logger.info("KSGitIngestionInfo with id {} exists: {}", ksGitIngestionInfoId,KSGitIngestionInfoExists);
@@ -199,12 +150,8 @@ public class DeletionService {
                 String ingestionStatus = ksGitInfo.getIngestionStatus();
                 logger.info("Ingestion Status is {}.", ingestionStatus);
 
-                List<VectorStore> vectorStoreMetadataDetails = null; /*vectorStoreGitDetailsExists
-                        ? vectorStoreRepository.findGitVectorByMetadata(ksDoctype,ksDocSource, ksFileSource, applicationName, ksBranch)
-                        : List.of();*/
-
                 if (KSGitInfoExists && KSGitIngestionInfoExists) {
-                    deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,vectorStoreMetadataDetails,applicationName);
+                    deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,applicationName);
 
                     String message = applicationName + " With Branch " + ksBranch + " records removed successfully having KSGitInfo with id "+ksGitInfoId;
                     logger.info(message);
@@ -224,13 +171,6 @@ public class DeletionService {
                         String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitIngestionInfo with id "+ksGitIngestionInfoId+" does not exist.";
                         logger.error(message);
 
-                        resultDTO.setSuccess(false);
-                        resultDTO.setMessage(message);
-                        simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
-                    } else if (vectorStoreMetadataDetails.isEmpty()) {
-                        String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to No VectorStore Data available";
-                        logger.error(message);
-
                         resultDTO.setSuccess(false);
                         resultDTO.setMessage(message);
                         simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
@@ -249,32 +189,21 @@ public class DeletionService {
         });
     }
 
-    private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,List<VectorStore> vectorStoreMetaDetails,String applicationName){
+    private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,String applicationName){
         try{
             switch (ingestionStatus){
                 case "INGESTION-ERROR":
                 case "INGESTION-IN-PROGRESS":
                 case "INGESTED":
-                    deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName);
-                    deleteVectorStores(vectorStoreMetaDetails,applicationName);
-                    break;
                 case "REPO-NEW":
                 case "REPO-CLONE-IN-PROGRESS":
                 case "REPO-CLONE-COMPLETED":
                 case "REPO-CLONE-FAILED":
-                    if (vectorStoreMetaDetails.isEmpty()) {
+                    deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName);
-                        deleteGitInfoAndIngestionInfo(ksGitInfoId, ksGitIngestionInfoId, applicationName);
-                    }else {
-                        // Throw a custom exception if vectorStoreMetaDetails is not empty
-                        throw new vectorStoreMetaDetailsEmptyException("VectorStoreMetaDetails is not empty for application name "+applicationName+" branch "+ksBranch+" and ingestion status is " + ingestionStatus);
-                    }
                     break;
                 default:
                     logger.warn("Unknown ingestion status: {}", ingestionStatus);
             }
-        } catch (vectorStoreMetaDetailsEmptyException e){
-            logger.error("vectorStoreMetaDetailsEmptyException occurred: ", e);
-            throw e;
         } catch (Exception e){
             logger.error("An error occurred while deleting records based on ingestion status: ", e);
             throw new RuntimeException("An error occurred while deleting records based on ingestion status", e);
@@ -293,45 +222,18 @@ public class DeletionService {
         }
     }
 
-    private void deleteVectorStores(List<VectorStore> vectorStoreMetadataDetails, String applicationName){
-        if(!vectorStoreMetadataDetails.isEmpty()){
-
-          /*  for (VectorStore store : vectorStoreMetadataDetails) {
-                String storeId=store.getId();
-                vectorStoreRepository.deleteById(storeId);
-                logger.info("VectorStore with id {} deleted successfully.", applicationName, storeId);
-            }*/
-        }
-    }
-
-
-
     @Async("asyncTaskExecutor")
     public void deleteTextRecords(String id) {
         try {
             boolean KSTextExists =  ksTextsRepository.existsById(id);
             
-            /*
+            if (KSTextExists) {
-            List<Object> vectorStoreMetadataDetails = vectorStoreRepository.findByKsInternalMainEntityId(id);
-
-            if (KSTextExists && !vectorStoreMetadataDetails.isEmpty()) {
-                for (VectorStore store : vectorStoreMetadataDetails) {
-                    vectorStoreRepository.deleteById(store.getId());
-                    logger.info("VectorStore with id {} deleted successfully.", store.getId()+" ");
-                }
-
                 ksTextsRepository.deleteById(id);
-                logger.info("KSDocument with id {} deleted successfully.", id+" ");
+                logger.info("KSDocument with id {} deleted successfully.", id);
-
-
                 logger.info("All records deleted successfully.");
             } else {
-                if (!KSTextExists) {
+                logger.warn("KSDocument with id {} does not exist.", id);
-                    logger.warn("KSDocument with id {} does not exist.", id+" ");
+            }
-                } else if (vectorStoreMetadataDetails.isEmpty()) {
-                    logger.warn("No VectorStore Data available",Thread.currentThread().getName());
-                }
-            }*/
         } catch (Exception e) {
             logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName());
             throw new RuntimeException("An error occurred while deleting records", e);

src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java

@@ -15,7 +15,6 @@ import java.util.regex.Pattern;
 
 import com.olympus.dto.ResultDTO;
 import com.olympus.apollo.exception.BranchCheckoutException;
-import com.olympus.apollo.repository.VectorStoreRepository;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.Repository;
@@ -25,7 +24,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.ai.document.Document;
 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
-import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.messaging.simp.SimpMessagingTemplate;
@@ -40,27 +38,18 @@ import com.olympus.apollo.repository.KSGitInfoRepository;
 @Service
 public class GitRepositoryIngestor {
 
-    private final VectorStore vectorStore;
-
     @Value("${gitlab.path}")
     private String basePath;
 
     @Autowired
     private KSGitInfoRepository ksGitInfoRepository;
 
-    @Autowired
-    private VectorStoreRepository vectorStoreRepository;
-
     @Autowired
     private GitService gitService;
 
     @Autowired
     private SimpMessagingTemplate simpMessagingTemplate;
 
-    public GitRepositoryIngestor(VectorStore vectorStore) {
-        this.vectorStore = vectorStore;
-    }
-
     Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
 
     @Async
@@ -147,8 +136,8 @@ public class GitRepositoryIngestor {
 
                     List<Document> splitDocuments = splitter.split(documents);
                     logger.info("Number of documents to be embedded: {}", splitDocuments.size());
-                    vectorStore.add(splitDocuments);
+                    // VectorStore operations removed
-                    logger.info("Documents embedded Successfully");
+                    logger.info("Documents processed successfully");
 
                 } catch (IOException e) {
                     ksGitInfo.setIngestionStatus("ERROR");
@@ -242,13 +231,7 @@ public class GitRepositoryIngestor {
                 default:
                     break;
             }
-            for (String fileToDelete : filePathsToDelete) {
+            // VectorStore operations removed for file deletions
-                Optional<com.olympus.model.apollo.VectorStore> optionalDocument = vectorStoreRepository.findByKsapplicationNameKsBranchFilePath(repoName,branchName,fileToDelete);
-                if (optionalDocument.isPresent()) {
-                    String vectorStoreId = optionalDocument.get().getId();
-                    vectorStoreRepository.deleteById(vectorStoreId);
-                }
-            }
         }
         gitService.checkOutRepository(repoName,branchName);
         String repoPath = basePath+ File.separator + repoName;
@@ -308,8 +291,8 @@ public class GitRepositoryIngestor {
 
                 List<Document> splitDocuments = splitter.split(documents);
                 logger.info("Number of documents: " + splitDocuments.size());
-                vectorStore.add(splitDocuments);
+                // VectorStore operations removed
-                logger.info("Documents embedded");
+                logger.info("Documents processed");
                 }
 
 

src/main/java/com/olympus/apollo/services/KSDocumentService.java

@@ -3,7 +3,15 @@ package com.olympus.apollo.services;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -16,9 +24,14 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Service;
 
+import com.olympus.apollo.client.NewDocumentServiceClient;
+import com.olympus.apollo.dto.DeleteDocumentResponse;
+import com.olympus.apollo.dto.KnowledgeSystemNode;
 import com.olympus.apollo.repository.KSDocumentRepository;
+import com.olympus.apollo.repository.KSVideoRepository;
 import com.olympus.apollo.security.entity.User;
 import com.olympus.model.apollo.KSDocument;
+import com.olympus.model.apollo.KSVideo;
 
 @Service
 public class KSDocumentService {
@@ -28,6 +41,12 @@ public class KSDocumentService {
     @Autowired
     private KSDocumentRepository ksdocRepo;
     
+    @Autowired
+    private KSVideoRepository ksVideoRepo;
+    
+    @Autowired
+    private NewDocumentServiceClient newDocumentServiceClient;
+
     public List<KSDocument> findByProjectNameAndApplicationName() {
         User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
 
@@ -68,4 +87,360 @@ public class KSDocumentService {
         return ResponseEntity.internalServerError().build();
     }
 }
+
+    /**
+     * Builds a filesystem-like structure based on ksKnowledgePath metadata for a given project
+     * Documents and videos without ksKnowledgePath in metadata are placed in the root folder
+     * @param projectName The name of the project to build the tree for
+     * @return Root node of the filesystem tree
+     */
+    public KnowledgeSystemNode buildKnowledgePathTree(String projectName) {
+        logger.info("Building knowledge path tree for project: " + projectName);
+        
+        try {
+            // Get all documents and videos for this project
+            List<KSDocument> documents = ksdocRepo.findAllByProjectName(projectName);
+            List<KSVideo> videos = ksVideoRepo.findByProjectName(projectName, Sort.by(Sort.Direction.DESC, "ingestionDate"));
+            
+            if ((documents == null || documents.isEmpty()) && (videos == null || videos.isEmpty())) {
+                logger.warn("No documents or videos found for project: " + projectName);
+                return null;
+            }
+            
+            // Create root node
+            KnowledgeSystemNode root = new KnowledgeSystemNode("root", "/", KnowledgeSystemNode.NodeType.FOLDER);
+            
+            // Build the tree structure
+            for (KSDocument doc : documents) {
+                String path = extractKnowledgePathFromMetadata(doc);
+                
+                // If no path in metadata, add document directly to root
+                if (path == null || path.trim().isEmpty()) {
+                    String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName();
+                    if (fileName != null && !fileName.isEmpty()) {
+                        KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.FILE);
+                        fileNode.setDocumentId(doc.getId());
+                        fileNode.setDocument(doc);
+                        root.addChild(fileNode);
+                    }
+                    continue;
+                }
+                
+                // Normalize path: remove leading/trailing slashes and split
+                path = path.trim();
+                if (path.startsWith("/")) {
+                    path = path.substring(1);
+                }
+                if (path.endsWith("/")) {
+                    path = path.substring(0, path.length() - 1);
+                }
+                
+                if (path.isEmpty()) {
+                    // Empty path after normalization, add to root
+                    String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName();
+                    if (fileName != null && !fileName.isEmpty()) {
+                        KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.FILE);
+                        fileNode.setDocumentId(doc.getId());
+                        fileNode.setDocument(doc);
+                        root.addChild(fileNode);
+                    }
+                    continue;
+                }
+                
+                String[] parts = path.split("/");
+                KnowledgeSystemNode currentNode = root;
+                StringBuilder currentPath = new StringBuilder();
+                
+                // Navigate/create folder structure for the entire path
+                // (ksKnowledgePath represents the folder where the document should be placed)
+                for (int i = 0; i < parts.length; i++) {
+                    String folderName = parts[i];
+                    currentPath.append("/").append(folderName);
+                    
+                    KnowledgeSystemNode childNode = findOrCreateChild(currentNode, folderName, 
+                        currentPath.toString(), KnowledgeSystemNode.NodeType.FOLDER);
+                    currentNode = childNode;
+                }
+                
+                // Add the file node using the document's actual fileName
+                String fileName = doc.getFileName() != null ? doc.getFileName() : doc.getName();
+                if (fileName != null && !fileName.isEmpty()) {
+                    currentPath.append("/").append(fileName);
+                    KnowledgeSystemNode fileNode = findOrCreateChild(currentNode, fileName, 
+                        currentPath.toString(), KnowledgeSystemNode.NodeType.FILE);
+                    fileNode.setDocumentId(doc.getId());
+                    fileNode.setDocument(doc);
+                }
+            }
+            
+            // Process videos with the same logic as documents
+            if (videos != null) {
+                for (KSVideo video : videos) {
+                    String path = extractKnowledgePathFromVideoMetadata(video);
+                    
+                    // If no path in metadata, add video directly to root
+                    if (path == null || path.trim().isEmpty()) {
+                        String fileName = video.getFileName() != null ? video.getFileName() : video.getName();
+                        if (fileName != null && !fileName.isEmpty()) {
+                            KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.VIDEO_FILE);
+                            fileNode.setVideoId(video.getId());
+                            fileNode.setVideo(video);
+                            root.addChild(fileNode);
+                        }
+                        continue;
+                    }
+                    
+                    // Normalize path: remove leading/trailing slashes and split
+                    path = path.trim();
+                    if (path.startsWith("/")) {
+                        path = path.substring(1);
+                    }
+                    if (path.endsWith("/")) {
+                        path = path.substring(0, path.length() - 1);
+                    }
+                    
+                    if (path.isEmpty()) {
+                        // Empty path after normalization, add to root
+                        String fileName = video.getFileName() != null ? video.getFileName() : video.getName();
+                        if (fileName != null && !fileName.isEmpty()) {
+                            KnowledgeSystemNode fileNode = new KnowledgeSystemNode(fileName, "/" + fileName, KnowledgeSystemNode.NodeType.VIDEO_FILE);
+                            fileNode.setVideoId(video.getId());
+                            fileNode.setVideo(video);
+                            root.addChild(fileNode);
+                        }
+                        continue;
+                    }
+                    
+                    String[] parts = path.split("/");
+                    KnowledgeSystemNode currentNode = root;
+                    StringBuilder currentPath = new StringBuilder();
+                    
+                    // Navigate/create folder structure for the entire path
+                    for (int i = 0; i < parts.length; i++) {
+                        String folderName = parts[i];
+                        currentPath.append("/").append(folderName);
+                        
+                        KnowledgeSystemNode childNode = findOrCreateChild(currentNode, folderName, 
+                            currentPath.toString(), KnowledgeSystemNode.NodeType.FOLDER);
+                        currentNode = childNode;
+                    }
+                    
+                    // Add the video file node using the video's actual fileName
+                    String fileName = video.getFileName() != null ? video.getFileName() : video.getName();
+                    if (fileName != null && !fileName.isEmpty()) {
+                        currentPath.append("/").append(fileName);
+                        KnowledgeSystemNode fileNode = findOrCreateChild(currentNode, fileName, 
+                            currentPath.toString(), KnowledgeSystemNode.NodeType.VIDEO_FILE);
+                        fileNode.setVideoId(video.getId());
+                        fileNode.setVideo(video);
+                    }
+                }
+            }
+            
+            return root;
+            
+        } catch (Exception e) {
+            logger.error("Error building knowledge path tree: " + e.getMessage(), e);
+            return null;
+        }
+    }
+    
+    /**
+     * Extracts ksKnowledgePath from document metadata
+     * @param doc The document to extract the path from
+     * @return The knowledge path or null if not present
+     */
+    private String extractKnowledgePathFromMetadata(KSDocument doc) {
+        if (doc.getIngestionInfo() == null) {
+            return null;
+        }
+        
+        Map<String, String> metadata = doc.getIngestionInfo().getMetadata();
+        if (metadata == null) {
+            return null;
+        }
+        
+        return metadata.get("ksKnowledgePath");
+    }
+    
+    /**
+     * Extracts ksKnowledgePath from video metadata
+     * @param video The video to extract the path from
+     * @return The knowledge path or null if not present
+     */
+    private String extractKnowledgePathFromVideoMetadata(KSVideo video) {
+        if (video.getIngestionInfo() == null) {
+            return null;
+        }
+        
+        HashMap<String, String> metadata = video.getIngestionInfo().getMetadata();
+        if (metadata == null) {
+            return null;
+        }
+        
+        return metadata.get("ksKnowledgePath");
+    }
+    
+    /**
+     * Helper method to find an existing child node or create a new one
+     */
+    private KnowledgeSystemNode findOrCreateChild(KnowledgeSystemNode parent, String name, 
+                                            String path, KnowledgeSystemNode.NodeType type) {
+        // Check if child already exists
+        for (KnowledgeSystemNode child : parent.getChildren()) {
+            if (child.getName().equals(name)) {
+                return child;
+            }
+        }
+        
+        // Create new child if not found
+        KnowledgeSystemNode newChild = new KnowledgeSystemNode(name, path, type);
+        parent.addChild(newChild);
+        return newChild;
+    }
+
+    /**
+     * Deletes a document from the database and filesystem by its ID
+     * This method will be extended to handle cancellation from the search index
+     * @param documentId The ID of the document to delete
+     * @return true if the document was deleted successfully, false if the document was not found
+     */
+    public boolean deleteDocument(String documentId) {
+        logger.info("Deleting document with ID: " + documentId);
+        
+        try {
+            // Check if document exists and retrieve it
+            KSDocument document = ksdocRepo.findById(documentId).orElse(null);
+            if (document == null) {
+                logger.warn("Document not found with ID: " + documentId);
+                return false;
+            }
+            
+            // Delete the physical file from filesystem
+            if (document.getFilePath() != null && !document.getFilePath().isEmpty()) {
+                try {
+                    Path filePath = Paths.get(document.getFilePath()).normalize();
+                    if (Files.exists(filePath)) {
+                        Files.delete(filePath);
+                        logger.info("Physical file deleted successfully: " + document.getFilePath());
+                    } else {
+                        logger.warn("Physical file not found at path: " + document.getFilePath());
+                    }
+                } catch (Exception fileException) {
+                    logger.error("Error deleting physical file: " + document.getFilePath() + " - " + fileException.getMessage(), fileException);
+                    // Continue with database deletion even if file deletion fails
+                }
+            }
+            
+            // Delete the document from the database
+            ksdocRepo.deleteById(documentId);
+            logger.info("Document deleted successfully from database with ID: " + documentId);
+            
+            // Delete from search index using the document ID
+            try {
+                // Extract project name from metadata
+                String projectName = "default";
+                if (document.getIngestionInfo() != null && document.getIngestionInfo().getMetadata() != null) {
+                    String metadataProjectName = document.getIngestionInfo().getMetadata().get("KsProjectName");
+                    if (metadataProjectName != null && !metadataProjectName.isEmpty()) {
+                        projectName = metadataProjectName;
+                    }
+                }
+                
+                ResponseEntity<DeleteDocumentResponse> response = newDocumentServiceClient.deleteDocumentByKsDocumentId(
+                    documentId,
+                    projectName,
+                    null // Use default Elasticsearch URL
+                );
+                
+                if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
+                    DeleteDocumentResponse deleteResponse = response.getBody();
+                    logger.info("Document deleted from search index. Deleted count: " + deleteResponse.getDeletedCount());
+                } else {
+                    logger.warn("Search index deletion returned non-successful status: " + response.getStatusCode());
+                }
+            } catch (Exception indexException) {
+                logger.error("Error deleting document from search index: " + indexException.getMessage(), indexException);
+                // Continue even if search index deletion fails - document is already deleted from DB
+            }
+            
+            return true;
+            
+        } catch (Exception e) {
+            logger.error("Error deleting document with ID " + documentId + ": " + e.getMessage(), e);
+            return false;
+        }
+    }
+    
+    /**
+     * Get all distinct metadata keys and their possible values from both KsDocument and KsVideo collections
+     * for a specific project
+     * 
+     * @param projectName The name of the project to query metadata for
+     * @return Map where keys are metadata field names and values are sets of all possible values for each field
+     */
+    public Map<String, Set<String>> getAllMetadataValues(String projectName) {
+        logger.info("Getting all metadata values for project: " + projectName);
+        
+        try {
+            // Map to store metadata keys and their distinct values
+            Map<String, Set<String>> metadataMap = new LinkedHashMap<>();
+            
+            // Get metadata from documents
+            List<KSDocument> documents = ksdocRepo.findAllMetadataByProjectName(projectName);
+            logger.info("Found " + documents.size() + " documents for project: " + projectName);
+            
+            for (KSDocument doc : documents) {
+                if (doc.getIngestionInfo() != null && doc.getIngestionInfo().getMetadata() != null) {
+                    HashMap<String, String> metadata = doc.getIngestionInfo().getMetadata();
+                    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+                        String key = entry.getKey();
+                        String value = entry.getValue();
+                        
+                        if (key != null && value != null && !value.trim().isEmpty()) {
+                            metadataMap.computeIfAbsent(key, k -> new HashSet<>()).add(value);
+                        }
+                    }
+                }
+            }
+            
+            // Get metadata from videos
+            List<KSVideo> videos = ksVideoRepo.findAllMetadataByProjectName(projectName);
+            logger.info("Found " + videos.size() + " videos for project: " + projectName);
+            
+            for (KSVideo video : videos) {
+                if (video.getIngestionInfo() != null && video.getIngestionInfo().getMetadata() != null) {
+                    HashMap<String, String> metadata = video.getIngestionInfo().getMetadata();
+                    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+                        String key = entry.getKey();
+                        String value = entry.getValue();
+                        
+                        if (key != null && value != null && !value.trim().isEmpty()) {
+                            metadataMap.computeIfAbsent(key, k -> new HashSet<>()).add(value);
+                        }
+                    }
+                }
+            }
+            
+            logger.info("Total unique metadata keys found: " + metadataMap.size());
+            
+            // Sort the values within each set for better presentation
+            Map<String, Set<String>> sortedMetadataMap = metadataMap.entrySet()
+                .stream()
+                .collect(Collectors.toMap(
+                    Map.Entry::getKey,
+                    entry -> entry.getValue().stream()
+                        .sorted()
+                        .collect(Collectors.toCollection(LinkedHashSet::new)),
+                    (e1, e2) -> e1,
+                    LinkedHashMap::new
+                ));
+            
+            return sortedMetadataMap;
+            
+        } catch (Exception e) {
+            logger.error("Error getting all metadata values for project " + projectName + ": " + e.getMessage(), e);
+            return new HashMap<>();
+        }
+    }
 }

src/main/java/com/olympus/apollo/services/KSIngestor.java

@@ -13,9 +13,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.ai.document.Document;
 import org.springframework.ai.reader.tika.TikaDocumentReader;
 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
-import org.springframework.ai.vectorstore.SearchRequest;
-import org.springframework.ai.vectorstore.SearchRequest.Builder;
-import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
@@ -47,9 +44,6 @@ public class KSIngestor {
     @Autowired
     private FileSystemStorageService storageService;
 
-    @Autowired
-    private VectorStore vectorStore;
-
     @Value("${ksingestor.embedded.doc.batch.size:20}")
     private int embDocsBatchSize;
 
@@ -60,14 +54,8 @@ public class KSIngestor {
     Logger logger = LoggerFactory.getLogger(KSIngestor.class);
 
     public void deleteAll(String document_file_name) {
-        Builder request_builder = SearchRequest.builder()
+        // VectorStore operations removed
-                .query("*")
+        logger.info("Document deletion requested for: " + document_file_name);
-                .similarityThreshold(0.0)
-                .filterExpression("'source'=='" + document_file_name + "'");
-        SearchRequest request = request_builder.build();
-        List<Document> docToDelete = vectorStore.similaritySearch(request);
-
-        logger.info("Number of documents to delete: " + docToDelete.size());
     }
 
     public IngestionOutput ingestLoop() {
@@ -202,9 +190,9 @@ public class KSIngestor {
                     splitDoc.getMetadata().putAll(metadata);
                 } 
                 
-                ksDocument.setIngestionMessage("Embedding documents");
+                ksDocument.setIngestionMessage("Processing documents");
                 ksDocumentRepository.save(ksDocument);
-                embedDocuments(splitDocs, ingestionInfo);
+                // VectorStore operations removed
             });
 
             ksDocument.setIngestionStatus("INGESTED");
@@ -274,7 +262,7 @@ public class KSIngestor {
                     splitDoc.getMetadata().putAll(meta2);
                     docIndex++;
                 }
-                embedtexts(splitDocs);
+                // VectorStore operations removed
 
             });
             //ksTexts.setIngestionStatus("INGESTED");
@@ -293,55 +281,28 @@ public class KSIngestor {
     }
 
     private void embedtexts(List<Document> docs) {
-
+        logger.info("Processing texts");
-        logger.info("Embedding texts");
-
         docs.forEach(doc -> logger.info("text metadata: " + doc.getMetadata()));
-        try {
+        // VectorStore operations removed
-            vectorStore.add(docs);
+        logger.info("Texts processed");
-            logger.info("Texts embedded");
-        } catch (Exception e) {
-            logger.error("Error embedding Texts: ", e);
-        }
     }
 
 
 
     private void embedDocuments(List<Document> docs, KSIngestionInfo ingestionInfo) {
-
+        logger.info("Processing documents");
-        logger.info("Embedding documents");
         int batchSize = embDocsBatchSize;
         for (int i = 0; i < docs.size(); i += batchSize) {
             int end = Math.min(i + batchSize, docs.size());
-            List<Document> currentList = docs.subList(i, end);
+            logger.info("Documents processed - Progress: Batch from {} to {} completed of {} total chunks", i, end, docs.size());
-            try {
-                Thread.sleep(embDocRetryTime);
-                vectorStore.add(currentList);
-                logger.info("Documents embedded - Progress: Batch from {} to {} completed of {} total chunks", i, end, docs.size());
-            } catch (Exception e) {
-                logger.error("Error embedding documents from {} to {}: {}", i, end, e.getMessage());
-            }
         }
-
+        // VectorStore operations removed
     }
 
     public List<Document> testSimilaritySearch(String query,String filterQuery) {
-        Builder request_builder = SearchRequest.builder()
+        // VectorStore operations removed
-        .query(query)
+        logger.info("Similarity search requested for query: " + query);
-        .topK(5)
+        return Collections.emptyList();
-        .similarityThreshold(0.1);
-
-        if(filterQuery != null && !filterQuery.isEmpty()){
-            request_builder.filterExpression(filterQuery);
-            logger.info("Using Filter expression: " + filterQuery);
-        } 
-
-        SearchRequest request  =  request_builder.build();
-        List<Document> docs  = vectorStore.similaritySearch(request);
-                
-        logger.info("Number of VDB retrieved documents: " + docs.size());
-
-        return docs;
     }
 
     public IngestionOutput setVideoInQueueIngestion(String id) {

src/main/resources/application.yml

@@ -25,14 +25,6 @@ spring:
         api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow"
     openai:
       api-key:  "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    vectorstore:
-      chroma:
-        client:
-          host: "http://108.142.36.18"
-          port: "80"
-          key-token: "BxZWXFXC4UMSxamf5xP5SioGIg3FPfP7"
-        initialize-schema: "true"
-        collection-name: "olympus"
   data:
     mongodb:
       uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000
@@ -70,3 +62,13 @@ java-re-module:
   url: "http://localhost:8084"
 
 tika.config: "tika-config.xml"
+new-document-service:
+  url: "http://localhost:8001"
+
+document:
+  storage:
+    path: /Users/andrea.terzani/Desktop/DEV/olympus/documents/apollo_storage/documents #C:\\repos\\olympus_ai\\documentStorage
+
+video:
+  storage:
+    path: /Users/andrea.terzani/Desktop/DEV/olympus/documents/apollo_storage/videos #C:\\repos\\olympus_ai\\videoStorage

start.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# ChromaDB Startup Script
+# This script starts a ChromaDB instance using Docker
+
+echo "Starting ChromaDB instance..."
+
+# Check if Docker is running
+if ! docker info > /dev/null 2>&1; then
+    echo "Error: Docker is not running. Please start Docker first."
+    exit 1
+fi
+
+# ChromaDB configuration
+CHROMA_PORT=8000
+CHROMA_CONTAINER_NAME="chromadb-olympus"
+CHROMA_IMAGE="ghcr.io/chroma-core/chroma:1.3.5"
+CHROMA_DATA_DIR="./chroma-data"
+
+# Create data directory if it doesn't exist
+mkdir -p "$CHROMA_DATA_DIR"
+
+# Check if container already exists
+if [ "$(docker ps -aq -f name=$CHROMA_CONTAINER_NAME)" ]; then
+    echo "ChromaDB container already exists."
+    
+    # Check if it's running
+    if [ "$(docker ps -q -f name=$CHROMA_CONTAINER_NAME)" ]; then
+        echo "ChromaDB is already running on port $CHROMA_PORT"
+        echo "Access it at: http://localhost:$CHROMA_PORT"
+        exit 0
+    else
+        echo "Starting existing ChromaDB container..."
+        docker start $CHROMA_CONTAINER_NAME
+    fi
+else
+    echo "Creating and starting new ChromaDB container..."
+    docker run -d \
+        --name $CHROMA_CONTAINER_NAME \
+        -p $CHROMA_PORT:8000 \
+        -v "$(pwd)/$CHROMA_DATA_DIR:/chroma/chroma" \
+        $CHROMA_IMAGE
+fi
+
+# Wait for ChromaDB to be ready
+echo "Waiting for ChromaDB to be ready..."
+sleep 3
+
+# Check if ChromaDB is responding (with timeout)
+if curl -s --max-time 5 http://localhost:$CHROMA_PORT/api/v1/heartbeat > /dev/null 2>&1; then
+    echo "✅ ChromaDB is up and running!"
+    echo "   - URL: http://localhost:$CHROMA_PORT"
+    echo "   - Data directory: $CHROMA_DATA_DIR"
+    echo "   - Container name: $CHROMA_CONTAINER_NAME"
+    echo ""
+    echo "To stop ChromaDB, run: docker stop $CHROMA_CONTAINER_NAME"
+    echo "To view logs, run: docker logs -f $CHROMA_CONTAINER_NAME"
+else
+    echo "⚠️  ChromaDB container started. Checking if it's ready..."
+    # Try without API endpoint - just check if container is running
+    if docker ps | grep -q $CHROMA_CONTAINER_NAME; then
+        echo "✅ ChromaDB container is running!"
+        echo "   - URL: http://localhost:$CHROMA_PORT"
+        echo "   - Data directory: $CHROMA_DATA_DIR"
+        echo "   - Container name: $CHROMA_CONTAINER_NAME"
+        echo "   - Note: API may take a few more seconds to be ready"
+        echo ""
+        echo "To stop ChromaDB, run: docker stop $CHROMA_CONTAINER_NAME"
+        echo "To view logs, run: docker logs -f $CHROMA_CONTAINER_NAME"
+    else
+        echo "❌ ChromaDB container failed to start"
+        echo "   Check logs with: docker logs $CHROMA_CONTAINER_NAME"
+        exit 1
+    fi
+fi
+
+echo ""
+echo "Update your application.yml with:"
+echo "  spring:"
+echo "    ai:"
+echo "      vectorstore:"
+echo "        chroma:"
+echo "          client:"
+echo "            host: \"http://localhost\""
+echo "            port: \"$CHROMA_PORT\""