admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Aggiunta della classe VectorSearchRequest e implementazione del controller SearchDocController per la ricerca vettoriale. Aggiornamento della configurazione di sicurezza e delle dipendenze nel pom.xml. Modifiche al file application.yml per la configurazione del servizio Azure.

Browse Source
This commit is contained in:
andrea.terzani
2024-11-26 11:59:29 +01:00
parent 88c59bfbef
commit 2bb0108bbc
8 changed files with 165 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pom.xml
View File

@@ -62,12 +62,12 @@
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-mongodb-atlas-store-spring-boot-starter</artifactId>
</dependency>
-->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-openai-spring-boot-starter</artifactId>
</dependency>
-->
<dependency>
<groupId>org.springframework.ai</groupId>
@@ -79,6 +79,11 @@
<artifactId>spring-ai-azure-store</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-chroma-store</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>

44
src/main/java/com/olympus/apollo/config/ApolloConfig.java Normal file
View File

@@ -0,0 +1,44 @@
package com.olympus.apollo.config;
import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.vectorstore.ChromaVectorStore;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
@Configuration
public class ApolloConfig {
private final OpenAiEmbeddingModel openAiEmbeddingModel;
private final AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel;
// Autowiring beans from both libraries
public ApolloConfig(OpenAiEmbeddingModel myServiceLib1,
AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel) {
this.openAiEmbeddingModel = myServiceLib1;
this.azureOpenAiEmbeddingModel = azureOpenAiEmbeddingModel;
}
// Re-declaring and marking one as primary
@Bean
public EmbeddingModel primaryMyService() {
return openAiEmbeddingModel; // Choose the one you want as primary
}
// Optionally declare the other bean without primary
@Bean
@Primary
public EmbeddingModel secondaryMyService() {
return azureOpenAiEmbeddingModel; // The other one
}
}

41
src/main/java/com/olympus/apollo/config/EmbeddingConfig.java → src/main/java/com/olympus/apollo/config/VectorStoreConfig.java
View File

@@ -1,30 +1,22 @@
package com.olympus.apollo.config;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.search.documents.indexes.SearchIndexClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel;
import org.springframework.ai.embedding.EmbeddingModel;
/*import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.ai.vectorstore.MongoDBAtlasVectorStore;*/
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.util.ArrayList;
import java.util.List;
@Configuration
public class EmbeddingConfig {
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.azure.AzureVectorStore;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.search.documents.indexes.SearchIndexClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
//@Configuration
public class VectorStoreConfig {
@Value("${spring.ai.vectorstore.azure.api-key}")
@@ -42,7 +34,7 @@ public class EmbeddingConfig {
}
@Bean
public VectorStore vectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) {
public VectorStore azureVectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) {
List<AzureVectorStore.MetadataField> fields = new ArrayList<>();
fields.add(AzureVectorStore.MetadataField.text("KsApplicationName"));
@@ -56,3 +48,6 @@ public class EmbeddingConfig {
}
}

53
src/main/java/com/olympus/apollo/controllers/SearchDocController.java Normal file
View File

@@ -0,0 +1,53 @@
package com.olympus.apollo.controllers;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.olympus.apollo.dto.VectorSearchRequest;
import ch.qos.logback.classic.Logger;
@RestController
@RequestMapping("/api/vsearch")
public class SearchDocController {
@Autowired
private VectorStore vectorStore;
Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class);
@PostMapping("/doc_search")
public List<String> vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) {
SearchRequest request = SearchRequest.defaults()
.withQuery(vectorSearchRequest.getQuery())
.withTopK(vectorSearchRequest.getTopK())
.withSimilarityThreshold(vectorSearchRequest.getThreshold());
if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){
request.withFilterExpression(vectorSearchRequest.getFilterQuery());
logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery());
}
List<Document> docs = this.vectorStore.similaritySearch(request);
logger.info("Number of VDB retrieved documents: " + docs.size());
List<String> results = new ArrayList<>();
for(Document doc : docs){
results.add(doc.getContent());
}
return results;
}
}

20
src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java Normal file
View File

@@ -0,0 +1,20 @@
package com.olympus.apollo.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class VectorSearchRequest {
private String query;
private String filterQuery;
private int topK;
private float threshold;
}

12
src/main/java/com/olympus/apollo/security/config/SecurityConfig.java
View File

@@ -1,21 +1,14 @@
package com.olympus.apollo.security.config;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.crypto.password.NoOpPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.SecurityFilterChain;
@@ -77,8 +70,9 @@ public class SecurityConfig {
http.csrf(csrf -> csrf.disable())
.sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.authorizeHttpRequests(auth -> auth.requestMatchers("/api/auth/**").permitAll()
.requestMatchers("/api/test/**").permitAll()
.anyRequest().authenticated()); //permitAll());
.requestMatchers("/api/test/**").permitAll()
.requestMatchers("/**").permitAll()
.anyRequest().authenticated()); //permitAll());
http.authenticationProvider(authenticationProvider());

1
src/main/java/com/olympus/apollo/services/KSIngestor.java
View File

@@ -232,6 +232,7 @@ public class KSIngestor {
}
public List<String> testSimilaritySearch(String query,String filterQuery) {
SearchRequest searchRequest = SearchRequest.defaults().withQuery(query).withTopK(5).withSimilarityThreshold(0.1);
if(filterQuery!=null && !filterQuery.isEmpty()){

26
src/main/resources/application.yml
View File

@@ -12,8 +12,8 @@ spring:
ai:
azure:
openai:
endpoint: "https://ai-olympus.openai.azure.com/"
api-key: "9fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
endpoint: "https://ai-olympus-new.openai.azure.com/"
api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow"
openai:
api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
vectorstore:
@@ -21,16 +21,25 @@ spring:
api-key: "jxKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
url: "https://search-olympus.search.windows.net"
initialize-schema: true
chroma:
client:
host: "http://108.142.74.161"
port: "8000"
key-token: "nVYLh3eq92aJP4x08dNdWngilPG2ooj9"
initialize-schema: "true"
collection-name: "olympus"
data:
mongodb:
uri:
database:
username:
password:
uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000
database: olympus
username: olympusadmin
password: Camilla123!
servlet:
multipart:
max-file-size: 5000000MB
max-request-size: 500000MB
main:
allow-circular-references: true
gitlab:
path: /mnt/apollo_storage/repository #C:\\repos\\olympus_ai\\gitClone
@@ -50,4 +59,7 @@ logging:
level:
root: INFO
#feign: DEBUG
#org.springframework.web.client: DEBUG
#org.springframework.web.client: DEBUG
java-re-module:
url: "http://localhost:8084"