admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved to mongo

Browse Source
This commit is contained in:
andrea.terzani
2024-07-29 12:41:29 +02:00
parent f93b20293c
commit 59ee7672c9
10 changed files with 99 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/com/olympus/apollo/controllers/KSFileController.java
View File

@@ -1,5 +1,9 @@
package com.olympus.apollo.controllers;
import java.util.HashMap;
import java.util.Map;
import org.codelibs.jhighlight.fastutil.Hash;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
@@ -39,19 +43,24 @@ public class KSFileController {
KSIngestionInfo ksIngestionInfo = new KSIngestionInfo();
ksIngestionInfo.setType("MD_DOCUMENT"); //TODO: This should be dynamic
ksIngestionInfo.setVdbIndex("atf_documentation");
ksIngestionInfo.setMetadata(filePath);
ksIngestionInfoRepository.save(ksIngestionInfo);
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicatioName","atf");
metadata.put("KsDoctype","documentation");
metadata.put("KsDoSource","wiki");
metadata.put("Source",file.getOriginalFilename());
ksIngestionInfo.setMetadata(metadata);
ksIngestionInfo.setDefaultChunkSize(1000);
ksIngestionInfo.setMinChunkSize(200);
ksIngestionInfo.setMaxNumberOfChunks(1000);
ksIngestionInfo.setMinChunkSizeToEmbed(20);
ksIngestionInfoRepository.save(ksIngestionInfo);
ksDocument.setIngestionInfo(ksIngestionInfo);
ksDocumentREpository.save(ksDocument);
return "OK";
}

65
src/main/java/com/olympus/apollo/models/KSDocument.java
View File

@@ -1,19 +1,19 @@
package com.olympus.apollo.models;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.Id;
import jakarta.persistence.OneToOne;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.Date;
import org.springframework.data.annotation.Id;
import lombok.Getter;
import lombok.Setter;
@Entity
@Document(collection = "ksdocuments")
@Getter @Setter
public class KSDocument {
@GeneratedValue
private @Id Long id;
private @Id String id;
private String name;
private String description;
@@ -21,54 +21,13 @@ public class KSDocument {
private String filePath;
private String fileName;
@OneToOne
private KSIngestionInfo ingestionInfo;
private String ingestionStatus;
private String ingestionMessage;
private Date ingestionDate;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getFilePath() {
return filePath;
}
public void setFilePath(String filePath) {
this.filePath = filePath;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public KSIngestionInfo getIngestionInfo() {
return ingestionInfo;
}
public void setIngestionInfo(KSIngestionInfo ingestionInfo) {
this.ingestionInfo = ingestionInfo;
}
public String getIngestionStatus() {
return ingestionStatus;
}
public void setIngestionStatus(String ingestionStatus) {
this.ingestionStatus = ingestionStatus;
}
private KSIngestionInfo ingestionInfo;
}

21
src/main/java/com/olympus/apollo/models/KSIngestionInfo.java
View File

@@ -1,31 +1,30 @@
package com.olympus.apollo.models;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.Id;
import java.util.HashMap;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Getter;
import lombok.Setter;
@Entity
@Document(collection = "ksingestioninfo")
@Getter @Setter
public class KSIngestionInfo {
@GeneratedValue
private @Id Long id;
private @Id String id;
private String ingestionMessage;
private String ingestionDate;
private String vdbIndex;
private String type;
private String metadata;
private HashMap<String,String> metadata;
private int minChunkSizeToEmbed;
private int maxNumberOfChunks;
private int minChunkSize;
private int defaultChunkSize;
}

3
src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java
View File

@@ -1,13 +1,12 @@
package com.olympus.apollo.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import com.olympus.apollo.models.KSDocument;
@Repository
public interface KSDocumentRepository extends CrudRepository<KSDocument, Long> {
public interface KSDocumentRepository extends CrudRepository<KSDocument, String> {
public Iterable<KSDocument> findAllByIngestionStatus(String status);
}

2
src/main/java/com/olympus/apollo/repository/KSIngestionInfoRepository.java
View File

@@ -6,6 +6,6 @@ import org.springframework.stereotype.Repository;
import com.olympus.apollo.models.KSIngestionInfo;
@Repository
public interface KSIngestionInfoRepository extends CrudRepository<KSIngestionInfo, Long> {
public interface KSIngestionInfoRepository extends CrudRepository<KSIngestionInfo, String> {
}

17
src/main/java/com/olympus/apollo/services/KSIngestor.java
View File

@@ -1,6 +1,7 @@
package com.olympus.apollo.services;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -19,6 +20,7 @@ import org.springframework.stereotype.Service;
import com.olympus.apollo.models.KSIngestionInfo;
import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository;
@Service
@@ -27,6 +29,8 @@ public class KSIngestor {
@Autowired
private KSDocumentRepository ksDocumentRepository;
@Autowired
private KSIngestionInfoRepository ksIngestionInfoRepository;
@Autowired
private FileSystemStorageService storageService;
@Autowired
@@ -77,8 +81,9 @@ public class KSIngestor {
}
embedDocuments(splitDocs, ingestionInfo);
});
ksDocument.setIngestionStatus("NEW");
ksDocument.setIngestionStatus("NEW");
ksDocument.setIngestionDate(new Date());
ksDocumentRepository.save(ksDocument);
});
@@ -111,15 +116,17 @@ public class KSIngestor {
private HashMap<String, String> getMetadata(KSIngestionInfo ingestionInfo) {
HashMap<String, String> metadata = new HashMap<String, String>();
String[] metadatas = ingestionInfo.getMetadata().split(";");
return ingestionInfo.getMetadata();
/* HashMap<String, String> metadata = new HashMap<String, String>();
for (String meta : metadatas) {
String[] keyValue = meta.split(":");
metadata.put(keyValue[0], keyValue[1]);
}
return metadata;
return metadata;*/
}

2
src/main/java/com/olympus/apollo/services/StorageProperties.java
View File

@@ -7,7 +7,7 @@ public class StorageProperties {
/**
* Folder location for storing files
*/
private String location = "/Users/andreaterzani/Desktop/dev/olympus/upload-dir";
private String location = "C:\\Users\\andrea.terzani\\dev\\olympus\\upload-dir";
public String getLocation() {
return location;