From 2bb0108bbca75af589e29cbeeca13b4a2993d2e0 Mon Sep 17 00:00:00 2001 From: "andrea.terzani" Date: Tue, 26 Nov 2024 11:59:29 +0100 Subject: [PATCH] Aggiunta della classe VectorSearchRequest e implementazione del controller SearchDocController per la ricerca vettoriale. Aggiornamento della configurazione di sicurezza e delle dipendenze nel pom.xml. Modifiche al file application.yml per la configurazione del servizio Azure. --- pom.xml | 9 +++- .../olympus/apollo/config/ApolloConfig.java | 44 +++++++++++++++ ...dingConfig.java => VectorStoreConfig.java} | 41 +++++++------- .../controllers/SearchDocController.java | 53 +++++++++++++++++++ .../apollo/dto/VectorSearchRequest.java | 20 +++++++ .../security/config/SecurityConfig.java | 12 ++--- .../olympus/apollo/services/KSIngestor.java | 1 + src/main/resources/application.yml | 26 ++++++--- 8 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 src/main/java/com/olympus/apollo/config/ApolloConfig.java rename src/main/java/com/olympus/apollo/config/{EmbeddingConfig.java => VectorStoreConfig.java} (72%) create mode 100644 src/main/java/com/olympus/apollo/controllers/SearchDocController.java create mode 100644 src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java diff --git a/pom.xml b/pom.xml index a1ba095..5e060a7 100644 --- a/pom.xml +++ b/pom.xml @@ -62,12 +62,12 @@ org.springframework.ai spring-ai-mongodb-atlas-store-spring-boot-starter - +--> org.springframework.ai spring-ai-openai-spring-boot-starter - --> + org.springframework.ai @@ -79,6 +79,11 @@ spring-ai-azure-store + + org.springframework.ai + spring-ai-chroma-store + + org.springframework.cloud spring-cloud-starter-netflix-eureka-client diff --git a/src/main/java/com/olympus/apollo/config/ApolloConfig.java b/src/main/java/com/olympus/apollo/config/ApolloConfig.java new file mode 100644 index 0000000..516bf18 --- /dev/null +++ b/src/main/java/com/olympus/apollo/config/ApolloConfig.java @@ -0,0 +1,44 @@ +package com.olympus.apollo.config; + +import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.openai.OpenAiEmbeddingModel; +import org.springframework.ai.vectorstore.ChromaVectorStore; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.azure.AzureVectorStore; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +public class ApolloConfig { + + private final OpenAiEmbeddingModel openAiEmbeddingModel; + private final AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel; + + + // Autowiring beans from both libraries + public ApolloConfig(OpenAiEmbeddingModel myServiceLib1, + AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel) { + this.openAiEmbeddingModel = myServiceLib1; + this.azureOpenAiEmbeddingModel = azureOpenAiEmbeddingModel; + + + } + + // Re-declaring and marking one as primary + @Bean + public EmbeddingModel primaryMyService() { + return openAiEmbeddingModel; // Choose the one you want as primary + } + + // Optionally declare the other bean without primary + @Bean + @Primary + public EmbeddingModel secondaryMyService() { + return azureOpenAiEmbeddingModel; // The other one + } + + + +} diff --git a/src/main/java/com/olympus/apollo/config/EmbeddingConfig.java b/src/main/java/com/olympus/apollo/config/VectorStoreConfig.java similarity index 72% rename from src/main/java/com/olympus/apollo/config/EmbeddingConfig.java rename to src/main/java/com/olympus/apollo/config/VectorStoreConfig.java index d8a2643..1a64a32 100644 --- a/src/main/java/com/olympus/apollo/config/EmbeddingConfig.java +++ b/src/main/java/com/olympus/apollo/config/VectorStoreConfig.java @@ -1,30 +1,22 @@ package com.olympus.apollo.config; -import com.azure.core.credential.AzureKeyCredential; -import com.azure.search.documents.indexes.SearchIndexClient; -import com.azure.search.documents.indexes.SearchIndexClientBuilder; -import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel; -import org.springframework.ai.embedding.EmbeddingModel; -/*import org.springframework.ai.openai.OpenAiEmbeddingModel; -import org.springframework.ai.openai.api.OpenAiApi; -import org.springframework.ai.vectorstore.MongoDBAtlasVectorStore;*/ - -import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.ai.vectorstore.azure.AzureVectorStore; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; - import java.util.ArrayList; import java.util.List; -@Configuration -public class EmbeddingConfig { +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.azure.AzureVectorStore; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.azure.core.credential.AzureKeyCredential; +import com.azure.search.documents.indexes.SearchIndexClient; +import com.azure.search.documents.indexes.SearchIndexClientBuilder; + +//@Configuration +public class VectorStoreConfig { @Value("${spring.ai.vectorstore.azure.api-key}") @@ -42,7 +34,7 @@ public class EmbeddingConfig { } @Bean - public VectorStore vectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) { + public VectorStore azureVectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) { List fields = new ArrayList<>(); fields.add(AzureVectorStore.MetadataField.text("KsApplicationName")); @@ -56,3 +48,6 @@ public class EmbeddingConfig { } } + + + diff --git a/src/main/java/com/olympus/apollo/controllers/SearchDocController.java b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java new file mode 100644 index 0000000..49392bf --- /dev/null +++ b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java @@ -0,0 +1,53 @@ +package com.olympus.apollo.controllers; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.LoggerFactory; +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.olympus.apollo.dto.VectorSearchRequest; + +import ch.qos.logback.classic.Logger; + +@RestController +@RequestMapping("/api/vsearch") + +public class SearchDocController { + + @Autowired + private VectorStore vectorStore; + + Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class); + + @PostMapping("/doc_search") + public List vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) { + + SearchRequest request = SearchRequest.defaults() + .withQuery(vectorSearchRequest.getQuery()) + .withTopK(vectorSearchRequest.getTopK()) + .withSimilarityThreshold(vectorSearchRequest.getThreshold()); + + if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){ + request.withFilterExpression(vectorSearchRequest.getFilterQuery()); + logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery()); + } + + List docs = this.vectorStore.similaritySearch(request); + logger.info("Number of VDB retrieved documents: " + docs.size()); + + List results = new ArrayList<>(); + for(Document doc : docs){ + results.add(doc.getContent()); + } + return results; + + } +} diff --git a/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java b/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java new file mode 100644 index 0000000..c3bea71 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java @@ -0,0 +1,20 @@ +package com.olympus.apollo.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class VectorSearchRequest { + + + private String query; + private String filterQuery; + private int topK; + private float threshold; + +} diff --git a/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java b/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java index 195b422..50e215f 100644 --- a/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java +++ b/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java @@ -1,21 +1,14 @@ package com.olympus.apollo.security.config; -import jakarta.servlet.http.HttpServletResponse; -import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import org.springframework.context.annotation.Lazy; import org.springframework.security.authentication.AuthenticationManager; - -import org.springframework.security.authentication.AuthenticationProvider; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.crypto.password.NoOpPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.SecurityFilterChain; @@ -77,8 +70,9 @@ public class SecurityConfig { http.csrf(csrf -> csrf.disable()) .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .authorizeHttpRequests(auth -> auth.requestMatchers("/api/auth/**").permitAll() - .requestMatchers("/api/test/**").permitAll() - .anyRequest().authenticated()); //permitAll()); + .requestMatchers("/api/test/**").permitAll() + .requestMatchers("/**").permitAll() + .anyRequest().authenticated()); //permitAll()); http.authenticationProvider(authenticationProvider()); diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 5463a7b..b277a71 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -232,6 +232,7 @@ public class KSIngestor { } public List testSimilaritySearch(String query,String filterQuery) { + SearchRequest searchRequest = SearchRequest.defaults().withQuery(query).withTopK(5).withSimilarityThreshold(0.1); if(filterQuery!=null && !filterQuery.isEmpty()){ diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6a638e2..be51639 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -12,8 +12,8 @@ spring: ai: azure: openai: - endpoint: "https://ai-olympus.openai.azure.com/" - api-key: "9fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + endpoint: "https://ai-olympus-new.openai.azure.com/" + api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" vectorstore: @@ -21,16 +21,25 @@ spring: api-key: "jxKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" url: "https://search-olympus.search.windows.net" initialize-schema: true + chroma: + client: + host: "http://108.142.74.161" + port: "8000" + key-token: "nVYLh3eq92aJP4x08dNdWngilPG2ooj9" + initialize-schema: "true" + collection-name: "olympus" data: mongodb: - uri: - database: - username: - password: + uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000 + database: olympus + username: olympusadmin + password: Camilla123! servlet: multipart: max-file-size: 5000000MB max-request-size: 500000MB + main: + allow-circular-references: true gitlab: path: /mnt/apollo_storage/repository #C:\\repos\\olympus_ai\\gitClone @@ -50,4 +59,7 @@ logging: level: root: INFO #feign: DEBUG - #org.springframework.web.client: DEBUG \ No newline at end of file + #org.springframework.web.client: DEBUG + +java-re-module: + url: "http://localhost:8084" \ No newline at end of file