From 3c93b9e6865685c929dff21e0ca1915679a27215 Mon Sep 17 00:00:00 2001 From: "DIR\\maria.del.valle" Date: Mon, 11 Nov 2024 17:29:06 +0100 Subject: [PATCH 1/2] added limit to embed documents --- .../olympus/apollo/services/KSIngestor.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 28e92ce..debe837 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -15,6 +15,7 @@ import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; @@ -38,6 +39,12 @@ public class KSIngestor { @Autowired private VectorStore vectorStore; + @Value("${ksingestor.embedded.doc.batch.size:20}") + private int embDocsBatchSize; + + @Value("${ksingestor.embedded.doc.retry.time:20000}") + private int embDocRetryTime; + Logger logger = LoggerFactory.getLogger(KSIngestor.class); @@ -209,12 +216,20 @@ public class KSIngestor { logger.info("Embedding documents"); docs.forEach(doc -> logger.info("Document metadata: " + doc.getMetadata())); - try { - vectorStore.add(docs); - logger.info("Documents embedded"); - } catch (Exception e) { - logger.error("Error embedding documents: ", e); + + int batchSize = embDocsBatchSize; + for (int i = 0; i < docs.size(); i += batchSize) { + int end = Math.min(i + batchSize, docs.size()); + List currentList = docs.subList(i, end); + try { + Thread.sleep(embDocRetryTime); + vectorStore.add(currentList); + logger.info("Documents embedded - Progress: Batch from {} to {} completed", i, end); + } catch (Exception e) { + logger.error("Error embedding documents from {} to {}: {}", i, end, e.getMessage()); + } } + } public List testSimilaritySearch(String query,String filterQuery) { From 840ae0ba51d8a5fc9a01c54c013c94812ba7b027 Mon Sep 17 00:00:00 2001 From: Florinda Date: Wed, 27 Nov 2024 09:27:52 +0100 Subject: [PATCH 2/2] Project + KsDocument --- .../FeApi/KsDocumentController.java | 46 ++++++++++- .../apollo/controllers/ProjectController.java | 45 +++++++++++ .../repository/KSDocumentRepository.java | 11 +++ .../apollo/repository/ProjectRepository.java | 12 +++ .../olympus/apollo/security/entity/User.java | 12 +++ .../apollo/services/KSDocumentService.java | 78 ++++++++++++++++++ .../apollo/services/ProjectService.java | 80 +++++++++++++++++++ src/main/resources/application.yml | 14 ++-- 8 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/olympus/apollo/controllers/ProjectController.java create mode 100644 src/main/java/com/olympus/apollo/repository/ProjectRepository.java create mode 100644 src/main/java/com/olympus/apollo/services/KSDocumentService.java create mode 100644 src/main/java/com/olympus/apollo/services/ProjectService.java diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java index b26ed49..ac01a41 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java @@ -3,10 +3,21 @@ package com.olympus.apollo.controllers.FeApi; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.UrlResource; import org.springframework.web.bind.annotation.*; import com.olympus.model.apollo.KSDocument; import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.services.KSDocumentService; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import java.nio.file.Files; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +import java.nio.file.Path; +import java.nio.file.Paths; + @RestController @RequestMapping("/fe-api/ksdocuments") @@ -14,12 +25,14 @@ public class KsDocumentController { @Autowired private KSDocumentRepository ksDocumentREpository; - + + @Autowired + private KSDocumentService ksDocumentService; @GetMapping("") public List getDocuments() { - List result = (List) ksDocumentREpository.findAll(); + List result = ksDocumentService.findByProjectNameAndApplicationName(); return result; } @@ -30,4 +43,33 @@ public class KsDocumentController { return result; } + + + /* @PostMapping("/downloadKSDocument") + public ResponseEntity downloadFile(@RequestBody KSDocument doc) { + try { + // Percorso al file + Path filePath = Paths.get(doc.getFilePath()).normalize(); + Resource resource = new UrlResource(filePath.toUri()); + + if (!resource.exists()) { + return ResponseEntity.notFound().build(); + } + + // Configurazione della risposta HTTP + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"") + .body(resource); + } catch (Exception e) { + return ResponseEntity.internalServerError().build(); + } + }*/ + + @PostMapping("/downloadKSDocument") +public ResponseEntity downloadFile(@RequestBody KSDocument doc) { + + return ksDocumentService.downloadKSDocument(doc); + +} + } diff --git a/src/main/java/com/olympus/apollo/controllers/ProjectController.java b/src/main/java/com/olympus/apollo/controllers/ProjectController.java new file mode 100644 index 0000000..5bf9474 --- /dev/null +++ b/src/main/java/com/olympus/apollo/controllers/ProjectController.java @@ -0,0 +1,45 @@ +package com.olympus.apollo.controllers; + +import java.util.Optional; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.apollo.services.ProjectService; +import com.olympus.model.Application; +import com.olympus.model.Project; + +@RestController +public class ProjectController { + + @Autowired + ProjectService projectService; + + @Autowired + ProjectRepository projectRepo; + + @GetMapping("/userProjects") + public Iterable getUserProjects() { + return projectService.getListProjectsByUser(); + } + + @PostMapping("/updateSelectedProject") + public User setSelectedProjects(@RequestBody Project projectId) { + return projectService.updateUserSelectedProject(projectId); + } + + @PostMapping("/updateSelectedApplication") + public User setSelectedApplication(@RequestBody Application application) { + return projectService.updateUserSelectedApplication(application); + } + @PostMapping("/getProject") + public Optional getUserProjects(@RequestBody String projectId) { + return projectRepo.findById(projectId); + } + + +} diff --git a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java index 3e07b81..1f86602 100644 --- a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java +++ b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java @@ -1,7 +1,11 @@ package com.olympus.apollo.repository; +import java.util.List; + +import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.data.rest.core.annotation.RepositoryRestResource; import org.springframework.web.bind.annotation.CrossOrigin; @@ -13,4 +17,11 @@ import com.olympus.model.apollo.KSDocument; public interface KSDocumentRepository extends MongoRepository { public Iterable findAllByIngestionStatus(String status); + + @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0, 'ingestionInfo.metadata.KsApplicationName': ?1 }") + public List findByProjectNameAndApplicationName(String projectName, String applicationName, Sort sort); + + @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0 }") + public List findByProjectName(String projectName, Sort sort); + } diff --git a/src/main/java/com/olympus/apollo/repository/ProjectRepository.java b/src/main/java/com/olympus/apollo/repository/ProjectRepository.java new file mode 100644 index 0000000..34f5371 --- /dev/null +++ b/src/main/java/com/olympus/apollo/repository/ProjectRepository.java @@ -0,0 +1,12 @@ +package com.olympus.apollo.repository; + +import org.bson.types.ObjectId; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; +import com.olympus.model.Project; + +@Repository +public interface ProjectRepository extends MongoRepository { + + +} diff --git a/src/main/java/com/olympus/apollo/security/entity/User.java b/src/main/java/com/olympus/apollo/security/entity/User.java index 0524cac..98763a5 100644 --- a/src/main/java/com/olympus/apollo/security/entity/User.java +++ b/src/main/java/com/olympus/apollo/security/entity/User.java @@ -4,10 +4,14 @@ import java.util.Collection; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.DocumentReference; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.userdetails.UserDetails; +import com.olympus.model.Application; +import com.olympus.model.Project; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -31,6 +35,14 @@ public class User implements UserDetails{ private String name; private String surname; + @DocumentReference + private Project selectedProject; + @DocumentReference + private List lstProjects; + + @DocumentReference + private Application selectedApplication; + private Role role; @Override diff --git a/src/main/java/com/olympus/apollo/services/KSDocumentService.java b/src/main/java/com/olympus/apollo/services/KSDocumentService.java new file mode 100644 index 0000000..d2481d5 --- /dev/null +++ b/src/main/java/com/olympus/apollo/services/KSDocumentService.java @@ -0,0 +1,78 @@ +package com.olympus.apollo.services; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.tomcat.util.openssl.openssl_h; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.model.apollo.KSDocument; + +@Service +public class KSDocumentService { + + private Logger logger = LoggerFactory.getLogger(KSDocumentService.class); + + @Autowired + private KSDocumentRepository ksdocRepo; + + public List findByProjectNameAndApplicationName() { + logger.info("findByProjectNameAndApplicationName function:"); + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + try { + if( principal.getSelectedApplication()==null){ + return ksdocRepo.findByProjectName(principal.getSelectedProject().getInternal_name(), Sort.by(Sort.Direction.DESC, "ingestionDate")); + }else{ + return ksdocRepo.findByProjectNameAndApplicationName(principal.getSelectedProject().getInternal_name(), principal.getSelectedApplication().getInternal_name(), Sort.by(Sort.Direction.DESC, "ingestionDate")); + } + } catch (Exception e) { + logger.error("Error in findByProjectNameAndApplicationName function: " + e.getMessage()); + } + + return null; + } + + public ResponseEntity downloadKSDocument(KSDocument doc) { + + logger.info("downloadKSDocument function:"); + + try { + // Percorso al file + Path filePath = Paths.get(doc.getFilePath()).normalize(); + Resource resource = new UrlResource(filePath.toUri()); + + if (!resource.exists()) { + return ResponseEntity.notFound().build(); + } + + // Determina il tipo MIME dinamicamente + String contentType = Files.probeContentType(filePath); + if (contentType == null) { + // Tipo MIME predefinito se non determinabile + contentType = "application/octet-stream"; + } + + // Configurazione della risposta HTTP + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_TYPE, contentType) // Tipo MIME dinamico + .body(resource); + } catch (Exception e) { + return ResponseEntity.internalServerError().build(); + } +} +} diff --git a/src/main/java/com/olympus/apollo/services/ProjectService.java b/src/main/java/com/olympus/apollo/services/ProjectService.java new file mode 100644 index 0000000..15ff917 --- /dev/null +++ b/src/main/java/com/olympus/apollo/services/ProjectService.java @@ -0,0 +1,80 @@ +package com.olympus.apollo.services; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.apollo.security.repository.UserRepository; +import com.olympus.model.Application; +import com.olympus.model.Project; + +@Service +public class ProjectService { + + private Logger logger = LoggerFactory.getLogger(ProjectService.class); + + @Autowired + private ProjectRepository projectRepo; + + @Autowired + private UserRepository userRepo; + + public List getListProjectsByUser(){ + logger.info("getListProjectByUser function:"); + + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + return principal.getLstProjects(); + + } + + public User updateUserSelectedProject(Project idProject){ + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + principal.setSelectedProject(idProject); + principal.setSelectedApplication(null); + User u = userRepo.save(principal); + return u; + + } + + public User updateUserSelectedApplication(Application application){ + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + principal.setSelectedApplication(application); + User u = userRepo.save(principal); + return u; + + } + + + public static List convertToObjectIdList(List listProjectIds) { + List objectIdList = new ArrayList<>(); + + for (String idString : listProjectIds) { + try { + // Crea un nuovo ObjectId dalla stringa e aggiungilo alla lista + objectIdList.add(new ObjectId(idString)); + } catch (IllegalArgumentException e) { + // Gestione degli errori per stringhe non valide che non possono essere convertite in ObjectId + System.err.println("Invalid ObjectId string: " + idString); + } + } + + return objectIdList; + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 9bbf31d..9e333ea 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -12,8 +12,8 @@ spring: ai: azure: openai: - endpoint: "https://ai-olympus.openai.azure.com/" - api-key: "9fb33cc69d914d4c8225b974876510b5" + endpoint: "https://ai-olympus-new.openai.azure.com/" + api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" vectorstore: @@ -23,10 +23,14 @@ spring: initialize-schema: true data: mongodb: - uri: mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus + uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000 database: olympus - username: olympus_adm - password: 26111979 + username: olympusadmin + password: Camilla123! + # uri: mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus + # database: olympus + # username: olympus_adm + # password: 26111979 servlet: multipart: max-file-size: 5000000MB