Aggiunta della classe VectorSearchRequest e implementazione del controller SearchDocController per la ricerca vettoriale. Aggiornamento della configurazione di sicurezza e delle dipendenze nel pom.xml. Modifiche al file application.yml per la configurazione del servizio Azure.

This commit is contained in:
andrea.terzani
2024-11-26 11:59:29 +01:00
parent 88c59bfbef
commit 2bb0108bbc
8 changed files with 165 additions and 41 deletions

View File

@@ -0,0 +1,44 @@
package com.olympus.apollo.config;
import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.vectorstore.ChromaVectorStore;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
@Configuration
public class ApolloConfig {
private final OpenAiEmbeddingModel openAiEmbeddingModel;
private final AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel;
// Autowiring beans from both libraries
public ApolloConfig(OpenAiEmbeddingModel myServiceLib1,
AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel) {
this.openAiEmbeddingModel = myServiceLib1;
this.azureOpenAiEmbeddingModel = azureOpenAiEmbeddingModel;
}
// Re-declaring and marking one as primary
@Bean
public EmbeddingModel primaryMyService() {
return openAiEmbeddingModel; // Choose the one you want as primary
}
// Optionally declare the other bean without primary
@Bean
@Primary
public EmbeddingModel secondaryMyService() {
return azureOpenAiEmbeddingModel; // The other one
}
}

View File

@@ -1,30 +1,22 @@
package com.olympus.apollo.config;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.search.documents.indexes.SearchIndexClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel;
import org.springframework.ai.embedding.EmbeddingModel;
/*import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.vectorstore.MongoDBAtlasVectorStore;*/
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.util.ArrayList;
import java.util.List;
@Configuration
public class EmbeddingConfig {
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.search.documents.indexes.SearchIndexClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
//@Configuration
public class VectorStoreConfig {
@Value("${spring.ai.vectorstore.azure.api-key}")
@@ -42,7 +34,7 @@ public class EmbeddingConfig {
}
@Bean
public VectorStore vectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) {
public VectorStore azureVectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) {
List<AzureVectorStore.MetadataField> fields = new ArrayList<>();
fields.add(AzureVectorStore.MetadataField.text("KsApplicationName"));
@@ -56,3 +48,6 @@ public class EmbeddingConfig {
}
}

View File

@@ -0,0 +1,53 @@
package com.olympus.apollo.controllers;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.olympus.apollo.dto.VectorSearchRequest;
import ch.qos.logback.classic.Logger;
@RestController
@RequestMapping("/api/vsearch")
public class SearchDocController {
@Autowired
private VectorStore vectorStore;
Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class);
@PostMapping("/doc_search")
public List<String> vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) {
SearchRequest request = SearchRequest.defaults()
.withQuery(vectorSearchRequest.getQuery())
.withTopK(vectorSearchRequest.getTopK())
.withSimilarityThreshold(vectorSearchRequest.getThreshold());
if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){
request.withFilterExpression(vectorSearchRequest.getFilterQuery());
logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery());
}
List<Document> docs = this.vectorStore.similaritySearch(request);
logger.info("Number of VDB retrieved documents: " + docs.size());
List<String> results = new ArrayList<>();
for(Document doc : docs){
results.add(doc.getContent());
}
return results;
}
}

View File

@@ -0,0 +1,20 @@
package com.olympus.apollo.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class VectorSearchRequest {
private String query;
private String filterQuery;
private int topK;
private float threshold;
}

View File

@@ -1,21 +1,14 @@
package com.olympus.apollo.security.config;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.crypto.password.NoOpPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.SecurityFilterChain;
@@ -77,8 +70,9 @@ public class SecurityConfig {
http.csrf(csrf -> csrf.disable())
.sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.authorizeHttpRequests(auth -> auth.requestMatchers("/api/auth/**").permitAll()
.requestMatchers("/api/test/**").permitAll()
.anyRequest().authenticated()); //permitAll());
.requestMatchers("/api/test/**").permitAll()
.requestMatchers("/**").permitAll()
.anyRequest().authenticated()); //permitAll());
http.authenticationProvider(authenticationProvider());

View File

@@ -232,6 +232,7 @@ public class KSIngestor {
}
public List<String> testSimilaritySearch(String query,String filterQuery) {
SearchRequest searchRequest = SearchRequest.defaults().withQuery(query).withTopK(5).withSimilarityThreshold(0.1);
if(filterQuery!=null && !filterQuery.isEmpty()){