Merge branch 'develop' into 'featureApollo'

# Conflicts:
#   src/main/java/com/olympus/apollo/controllers/TestController.java
#   src/main/java/com/olympus/apollo/services/StorageProperties.java
#   src/main/resources/application.properties
This commit is contained in:
Sumedh
2024-08-09 03:10:43 +00:00
29 changed files with 815 additions and 72 deletions

View File

@@ -0,0 +1,52 @@
package com.olympus.apollo.controllers.FeApi;
import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreDetails;
import com.olympus.apollo.models.VectorStore;
import com.olympus.apollo.repository.VectorStoreRepository;
import com.olympus.apollo.services.DeletionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
@RequestMapping("/fe-api/vector-store")
@CrossOrigin(origins = "http://localhost:5173")
public class VectorStoreController {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Autowired
private DeletionService deletionService;
@GetMapping("")
public List<VectorStore> getAllVectorStore(){
List<VectorStore> result = (List<VectorStore>) vectorStoreRepository.findAll();
return result;
}
@GetMapping("/{id}")
public VectorStore getVectorData(@PathVariable String id){
VectorStore result = vectorStoreRepository.findById(id).get();
return result;
}
@GetMapping("/ids")
public List<String> getAllVectorStoreIds(){
List<String> ids = vectorStoreRepository.findAllIds();
return ids;
}
@GetMapping("/details")
public List<VectorStoreDetails> getAllVectorStoreDetails(){
return vectorStoreRepository.findAllDetails();
}
@PostMapping("/deleteRecords")
public String deleteRecords(@RequestBody DeletionRequest deletionRequest){
deletionService.deleteRecords(deletionRequest);
return "Records Deleted Successfully";
}
}

View File

@@ -1,17 +1,13 @@
package com.olympus.apollo.controllers;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Date;
import java.text.SimpleDateFormat;
import org.codelibs.jhighlight.fastutil.Hash;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import com.olympus.apollo.models.KSDocument;
import com.olympus.apollo.models.KSIngestionInfo;
@@ -19,7 +15,7 @@ import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.services.StorageFileNotFoundException;
import com.olympus.apollo.services.StorageService;
import com.olympus.apollo.models.FileUploadDTO;
import com.olympus.apollo.dto.FileUploadDTO;
@CrossOrigin
@RestController
public class KSFileController {
@@ -44,6 +40,7 @@ public class KSFileController {
ksDocument.setName(file.getOriginalFilename());
ksDocument.setDescription(fileUploadDTO.getDescription());
ksDocument.setIngestionStatus("NEW");
ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
Date now = new Date();
ksDocument.setIngestionDate(now);
@@ -55,8 +52,8 @@ public class KSFileController {
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", fileUploadDTO.getKsApplicationName());
metadata.put("KsDoctype", fileUploadDTO.getKsDocType());
metadata.put("KsDocSource", fileUploadDTO.getKsDocType());
metadata.put("Source", file.getOriginalFilename());
metadata.put("KsDocSource", fileUploadDTO.getKsDocSource());
metadata.put("KsFileSource", file.getOriginalFilename());
ksIngestionInfo.setMetadata(metadata);
ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize());

View File

@@ -0,0 +1,15 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Getter @Setter
public class DeletionRequest {
private String ksDocumentId;
private String ksIngestionInfoId;
private String ksDoctype;
private String ksDocSource;
private String ksFileSource;
private String ksApplicationName;
}

View File

@@ -1,4 +1,4 @@
package com.olympus.apollo.models;
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;

View File

@@ -15,6 +15,4 @@ public class IngestionOutput {
public IngestionOutput(){
ingestedDocumentId=new ArrayList<String>();
}
}

View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Getter @Setter
public class VectorStoreDetails {
private String id;
private VectorStoreMetadataDetails metadata;
}

View File

@@ -0,0 +1,14 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Getter @Setter
public class VectorStoreMetadataDetails {
private String KsApplicationName;
private String KsDoctype;
private String KsDocSource;
private String KsFileSource;
}

View File

@@ -25,6 +25,7 @@ public class KSDocument {
private String ingestionStatus;
//private String ingestionMessage;
private Date ingestionDate;
private String ingestionDateFormat;
private KSIngestionInfo ingestionInfo;

View File

@@ -0,0 +1,20 @@
package com.olympus.apollo.models;
import lombok.Getter;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.HashMap;
@Document(collection = "vector_store")
@Getter
public class VectorStore {
private @Id String id;
private HashMap<String, String> metadata;
private String content;
private String media;
private String embedding;
private HashMap<String, String> contentFormatter;
}

View File

@@ -0,0 +1,27 @@
package com.olympus.apollo.repository;
import com.olympus.apollo.dto.VectorStoreDetails;
import com.olympus.apollo.models.VectorStore;
import org.springframework.data.mongodb.repository.Aggregation;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface VectorStoreRepository extends MongoRepository<VectorStore, String> {
@Aggregation(pipeline = {
"{ $project: { _id: 1 } }"
})
List<String> findAllIds();
@Aggregation(pipeline = {
"{ $project: { _id: 1, metadata: 1 } }"
})
List<VectorStoreDetails> findAllDetails();
@Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}")
List<VectorStore> findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
}

View File

@@ -0,0 +1,17 @@
package com.olympus.apollo.security.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@Configuration
public class CorsConfig implements WebMvcConfigurer {
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**")
.allowedOrigins("*")
.allowedHeaders("*")
.allowedMethods("GET", "POST", "PUT", "DELETE");
}
}

View File

@@ -0,0 +1,89 @@
package com.olympus.apollo.security.config;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.crypto.password.NoOpPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import com.olympus.apollo.security.filter.JwtTokenFilter;
import com.olympus.apollo.security.services.CustomUserDetailsService;
@EnableWebSecurity
@Configuration
public class SecurityConfig {
@Autowired
CustomUserDetailsService userDetailsService;
/* @Autowired
private AuthEntryPointJwt unauthorizedHandler;
*/
@Bean
public JwtTokenFilter authenticationJwtTokenFilter() {
return new JwtTokenFilter();
}
//@Override
//public void configure(AuthenticationManagerBuilder authenticationManagerBuilder) throws Exception {
// authenticationManagerBuilder.userDetailsService(userDetailsService).passwordEncoder(passwordEncoder());
//}
@Bean
public DaoAuthenticationProvider authenticationProvider() {
DaoAuthenticationProvider authProvider = new DaoAuthenticationProvider();
authProvider.setUserDetailsService(userDetailsService);
authProvider.setPasswordEncoder(passwordEncoder());
return authProvider;
}
//@Bean
//@Override
//public AuthenticationManager authenticationManagerBean() throws Exception {
// return super.authenticationManagerBean();
//}
@Bean
public AuthenticationManager authenticationManager(AuthenticationConfiguration authConfig) throws Exception {
return authConfig.getAuthenticationManager();
}
@Bean
PasswordEncoder passwordEncoder() {
return NoOpPasswordEncoder.getInstance();
}
@Bean
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
http.csrf(csrf -> csrf.disable())
.sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.authorizeHttpRequests(auth -> auth.requestMatchers("/api/auth/**").permitAll().requestMatchers("/api/test/**")
.permitAll().anyRequest().authenticated());
http.authenticationProvider(authenticationProvider());
http.addFilterBefore(authenticationJwtTokenFilter(), UsernamePasswordAuthenticationFilter.class);
return http.build();
}
}

View File

@@ -0,0 +1,90 @@
package com.olympus.apollo.security.controllers;
import java.security.Principal;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.olympus.apollo.security.dto.AuthenticationRequest;
import com.olympus.apollo.security.dto.AuthenticationResponse;
import com.olympus.apollo.security.dto.FetchUserResponse;
import com.olympus.apollo.security.entity.User;
import com.olympus.apollo.security.utility.JwtTokenProvider;
@RestController
@CrossOrigin
@RequestMapping("/api/auth")
public class AuthController {
@Autowired
private AuthenticationManager authenticationManager;
@Autowired
private JwtTokenProvider jwtTokenProvider;
@PostMapping("/login")
public ResponseEntity<?> authenticateUser(@RequestBody AuthenticationRequest authenticationRequest) {
Authentication authentication = authenticationManager.authenticate(
new UsernamePasswordAuthenticationToken(
authenticationRequest.getUsername(),
authenticationRequest.getPassword()
)
);
SecurityContextHolder.getContext().setAuthentication(authentication);
String jwt = jwtTokenProvider.createToken(authentication);
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.add("authorization", jwt);
httpHeaders.add("access-control-expose-headers", "authorization");
AuthenticationResponse authenticationResponse = new AuthenticationResponse(jwt, (User) authentication.getPrincipal());
return ResponseEntity.ok().headers(httpHeaders).body(authenticationResponse);
}
@GetMapping("/fetch-user")
public FetchUserResponse fetchUser(Authentication authentication) {
User principal = (User) authentication.getPrincipal();
principal.setPassword(null);
FetchUserResponse fetchUserResponse = new FetchUserResponse();
fetchUserResponse.setData(principal);
return fetchUserResponse;
}
@GetMapping("/refresh-token")
public ResponseEntity<?> refreshToken(Authentication authentication) {
SecurityContextHolder.getContext().setAuthentication(authentication);
String jwt = jwtTokenProvider.createToken(authentication);
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.add("authorization", jwt);
httpHeaders.add("access-control-expose-headers", "authorization");
AuthenticationResponse authenticationResponse = new AuthenticationResponse(jwt, (User) authentication.getPrincipal());
return ResponseEntity.ok().headers(httpHeaders).body(authenticationResponse);
}
@GetMapping("/test")
public ResponseEntity<?> test() {
return ResponseEntity.ok(" you have access now ");
}
}

View File

@@ -0,0 +1,16 @@
package com.olympus.apollo.security.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@AllArgsConstructor
@NoArgsConstructor
@Getter
@Setter
public class AuthenticationRequest {
private String username;
private String password;
}

View File

@@ -0,0 +1,19 @@
package com.olympus.apollo.security.dto;
import com.olympus.apollo.security.entity.User;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class AuthenticationResponse {
private String accessToken;
private User data;
}

View File

@@ -0,0 +1,23 @@
package com.olympus.apollo.security.dto;
import java.util.Collection;
import java.util.List;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
import com.olympus.apollo.security.entity.Role;
import com.olympus.apollo.security.entity.User;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FetchUserResponse {
private User data;
}

View File

@@ -0,0 +1,5 @@
package com.olympus.apollo.security.entity;
public enum Role {
ADMIN,USER
}

View File

@@ -0,0 +1,43 @@
package com.olympus.apollo.security.entity;
import java.util.Collection;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.userdetails.UserDetails;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.List;
@Document(collection = "users")
@Getter @Setter
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class User implements UserDetails{
@Id
private String id;
private String username;
private String password;
private String name;
private String surname;
private Role role;
@Override
public Collection<? extends GrantedAuthority> getAuthorities() {
return List.of(new SimpleGrantedAuthority(role.name()));
}
}

View File

@@ -0,0 +1,53 @@
package com.olympus.apollo.security.filter;
import java.io.IOException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.web.authentication.WebAuthenticationDetailsSource;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import com.olympus.apollo.security.utility.JwtTokenProvider;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
@Component
@AllArgsConstructor
@NoArgsConstructor
public class JwtTokenFilter extends OncePerRequestFilter {
@Autowired
private JwtTokenProvider jwtTokenProvider;
@Autowired
private UserDetailsService userDetailsService;
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException {
String token = jwtTokenProvider.resolveToken(request);
if (token != null && jwtTokenProvider.validateToken(token)) {
UserDetails userDetails = userDetailsService.loadUserByUsername(jwtTokenProvider.getUsername(token));
UsernamePasswordAuthenticationToken authentication = new UsernamePasswordAuthenticationToken(userDetails, null, userDetails.getAuthorities());
authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(request));
SecurityContextHolder.getContext().setAuthentication(authentication);
}
filterChain.doFilter(request, response);
}
}

View File

@@ -0,0 +1,14 @@
package com.olympus.apollo.security.repository;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import com.olympus.apollo.security.entity.User;
@Repository
public interface UserRepository extends MongoRepository<User, String> {
@Query("{username:'?0'}")
User findUserByUsername(String username);
}

View File

@@ -0,0 +1,35 @@
package com.olympus.apollo.security.services;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.stereotype.Service;
import com.olympus.apollo.security.entity.User;
import com.olympus.apollo.security.repository.UserRepository;
@Configuration
@Service
public class CustomUserDetailsService implements UserDetailsService {
@Autowired
private UserRepository userRepository;
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
try {
User user = userRepository.findUserByUsername(username);
if(user != null){
return user;
}else{
throw new Exception("user Not found ");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -0,0 +1,29 @@
package com.olympus.apollo.security.utility;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.web.authentication.www.BasicAuthenticationEntryPoint;
import org.springframework.stereotype.Component;
import io.jsonwebtoken.io.IOException;
@Component
public class JwtAuthenticationEntryPoint extends BasicAuthenticationEntryPoint {
@Override
public void commence(HttpServletRequest request, HttpServletResponse response,
AuthenticationException authException) throws IOException, java.io.IOException {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
response.setContentType("application/json");
response.getWriter().write("{ \"message\": \"" + authException.getMessage() + "\" }");
}
@Override
public void afterPropertiesSet() {
setRealmName("JWT Authentication");
super.afterPropertiesSet();
}
}

View File

@@ -0,0 +1,76 @@
package com.olympus.apollo.security.utility;
import org.springframework.stereotype.Component;
import io.jsonwebtoken.*;
import io.jsonwebtoken.security.Keys;
import io.jsonwebtoken.security.SignatureException;
import jakarta.servlet.http.HttpServletRequest;
import lombok.extern.slf4j.Slf4j;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.security.Key;
import java.util.Date;
@Component
@Slf4j
public class JwtTokenProvider {
Key key = Keys.secretKeyFor(SignatureAlgorithm.HS512);
public String createToken(Authentication authentication) {
UserDetails userDetails = (UserDetails) authentication.getPrincipal();
Date now = new Date();
Date expiryDate = new Date(now.getTime() + 3600000);
return Jwts.builder()
.setSubject(userDetails.getUsername())
.setIssuedAt(new Date())
.setExpiration(expiryDate)
.signWith(SignatureAlgorithm.HS512, key)
.compact();
}
public String resolveToken(HttpServletRequest request) {
String bearerToken = request.getHeader("Authorization");
if (StringUtils.hasText(bearerToken) && bearerToken.startsWith("Bearer ")) {
return bearerToken.substring(7);
}
return null;
}
// Check if the token is valid and not expired
public boolean validateToken(String token) {
try {
Jwts.parser().setSigningKey(key).parseClaimsJws(token);
return true;
} catch (MalformedJwtException ex) {
log.error("Invalid JWT token");
} catch (ExpiredJwtException ex) {
log.error("Expired JWT token");
} catch (UnsupportedJwtException ex) {
log.error("Unsupported JWT token");
} catch (IllegalArgumentException ex) {
log.error("JWT claims string is empty");
} catch (SignatureException e) {
log.error("there is an error with the signature of you token ");
}
return false;
}
// Extract the username from the JWT token
public String getUsername(String token) {
return Jwts.parser()
.setSigningKey(key)
.parseClaimsJws(token)
.getBody()
.getSubject();
}
}

View File

@@ -0,0 +1,70 @@
package com.olympus.apollo.services;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore;
import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.repository.VectorStoreRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class DeletionService {
private static final Logger logger = LoggerFactory.getLogger(DeletionService.class);
@Autowired
private KSDocumentRepository ksDocumentRepository;
@Autowired
private KSIngestionInfoRepository ksIngestionInfoRepository;
@Autowired
private VectorStoreRepository vectorStoreRepository;
public void deleteRecords(DeletionRequest deletionRequest) {
try {
boolean KSDocumentExists = deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty() && ksDocumentRepository.existsById(deletionRequest.getKsDocumentId());
boolean KSIngestionInfoExists = deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty() && ksIngestionInfoRepository.existsById(deletionRequest.getKsIngestionInfoId());
boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null;
List<VectorStore> vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of();
if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) {
if (deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty()) {
ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId());
logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId());
}
if (deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty()) {
ksIngestionInfoRepository.deleteById(deletionRequest.getKsIngestionInfoId());
logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", store.getId());
}
logger.info("All records deleted successfully.");
} else {
if (!KSDocumentExists) {
logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId());
} else if (!KSIngestionInfoExists) {
logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
}
}
} catch (Exception e) {
logger.error("An error occurred while deleting records: ", e);
throw new RuntimeException("An error occurred while deleting records", e);
}
}
}

View File

@@ -1,6 +1,7 @@
package com.olympus.apollo.services;
import java.util.*;
import java.text.SimpleDateFormat;
import com.olympus.apollo.dto.IngestionOutput;
import com.olympus.apollo.models.KSDocument;
@@ -82,6 +83,7 @@ public class KSIngestor {
});
ksDocument.setIngestionStatus("INGESTED");//we have to set to DONE
ksDocument.setIngestionDate(new Date());
ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
ksDocumentRepository.save(ksDocument);
@@ -144,7 +146,7 @@ public class KSIngestor {
});
ksDocument.setIngestionStatus("INGESTED");
ksDocument.setIngestionDate(new Date());
ksDocument.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
ksDocumentRepository.save(ksDocument);
ingestionLoopOutput.getIngestedDocumentId().add(ksDocument.getId());
@@ -172,12 +174,12 @@ public class KSIngestor {
}
}
public List<String> testSimilaritySearch(String query,String filter_doc_type) {
public List<String> testSimilaritySearch(String query,String filterQuery) {
List<Document> docs = vectorStore.similaritySearch(
SearchRequest.defaults()
.withQuery(query)
.withTopK(5).withSimilarityThreshold(0.8)
.withFilterExpression("'KsDoctype'=='"+filter_doc_type+"'"));
.withFilterExpression(filterQuery));
List<String> result = new ArrayList<String>();
for (Document doc : docs) {

View File

@@ -4,7 +4,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties("storage")
public class StorageProperties {
/**
/**
* Folder location for storing files
*/
private String location = "C:\\Users\\vinayak.c.mishra\\dev\\olympus\\upload-dir";