Merged PR 68: codegenie fix bugs
This commit is contained in:
@@ -19,20 +19,30 @@ public interface ScenarioRepository extends MongoRepository<Scenario, String> {
|
|||||||
List<Scenario> findByUsableFor(String projectId);*/
|
List<Scenario> findByUsableFor(String projectId);*/
|
||||||
|
|
||||||
//List<Scenario> findByAvailableForProjects_Id(String projectId);
|
//List<Scenario> findByAvailableForProjects_Id(String projectId);
|
||||||
List<Scenario> findByAvailableForProjects_IdAndVisible(String projectId, String visible);
|
//List<Scenario> findByAvailableForProjects_IdAndVisible(String projectId, String visible);
|
||||||
|
List<Scenario> findByAvailableForProjects_IdAndVisibleIn(String projectId, List<String> visible);
|
||||||
|
|
||||||
|
List<Scenario> findByAvailableForApplications_IdAndVisibleIn(String projectId, List<String> visible);
|
||||||
|
|
||||||
|
List<Scenario> findByAvailableForProjectsIsNullAndAvailableForApplicationsIsNullAndVisibleIn(List<String> visible);
|
||||||
|
|
||||||
|
|
||||||
List<Scenario> findByAvailableForApplications_IdAndVisible(String projectId, String visible);
|
/* @Query("{ 'visible': 'Y', 'category': 'RE', " +
|
||||||
|
|
||||||
List<Scenario> findByAvailableForProjectsIsNullAndAvailableForApplicationsIsNullAndVisible(String visible);
|
|
||||||
|
|
||||||
|
|
||||||
@Query("{ 'visible': 'Y', 'category': 'RE', " +
|
|
||||||
" $or: [ " +
|
" $or: [ " +
|
||||||
" { 'availableForProjects': ObjectId(?0) }, " +
|
" { 'availableForProjects': ObjectId(?0) }, " +
|
||||||
" { 'availableForApplications': ObjectId(?1) } " +
|
" { 'availableForApplications': ObjectId(?1) } " +
|
||||||
" ] " +
|
" ] " +
|
||||||
"}")
|
"}")
|
||||||
List<Scenario> findByVisibleAndCategoryAndProjectOrApplication(String projectId, String applicationId);
|
List<Scenario> findByVisibleAndCategoryAndProjectOrApplication(String projectId, String applicationId);
|
||||||
|
*/
|
||||||
|
|
||||||
|
@Query("{ 'visible': { $in: ?2 }, 'category': 'RE', " +
|
||||||
|
" $or: [ " +
|
||||||
|
" { 'availableForProjects': ObjectId(?0) }, " +
|
||||||
|
" { 'availableForApplications': ObjectId(?1) } " +
|
||||||
|
" ] " +
|
||||||
|
"}")
|
||||||
|
List<Scenario> findByVisibleInAndCategoryAndProjectOrApplication(String projectId, String applicationId, List<String> visibleValues);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,8 +27,18 @@ public class ScenarioService {
|
|||||||
public List<Scenario> getListScenariosByProject(String project){
|
public List<Scenario> getListScenariosByProject(String project){
|
||||||
logger.info("getListProjectByUser function:");
|
logger.info("getListProjectByUser function:");
|
||||||
List<Scenario> lstScenarios = null;
|
List<Scenario> lstScenarios = null;
|
||||||
|
|
||||||
|
List<String> scenAccept = new ArrayList<String>();
|
||||||
|
scenAccept.add("Y");
|
||||||
|
|
||||||
try{
|
try{
|
||||||
lstScenarios = scenarioRepo.findByAvailableForProjects_IdAndVisible(project, "Y");
|
|
||||||
|
User user = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
|
||||||
|
if(user.getRole().name().equals("ADMIN")){
|
||||||
|
scenAccept.add("DRAFT");
|
||||||
|
}
|
||||||
|
lstScenarios = scenarioRepo.findByAvailableForProjects_IdAndVisibleIn(project, scenAccept);
|
||||||
|
//lstScenarios = scenarioRepo.findByAvailableForProjects_IdAndVisible(project, "Y");
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
logger.error("Exception ScenarioRepository:", e.getMessage());
|
logger.error("Exception ScenarioRepository:", e.getMessage());
|
||||||
}
|
}
|
||||||
@@ -41,8 +51,16 @@ public class ScenarioService {
|
|||||||
public List<Scenario> getListScenariosByApplication(String app){
|
public List<Scenario> getListScenariosByApplication(String app){
|
||||||
logger.info("getListProjectByUser function:");
|
logger.info("getListProjectByUser function:");
|
||||||
List<Scenario> lstScenarios = null;
|
List<Scenario> lstScenarios = null;
|
||||||
|
List<String> scenAccept = new ArrayList<String>();
|
||||||
|
scenAccept.add("Y");
|
||||||
try{
|
try{
|
||||||
lstScenarios = scenarioRepo.findByAvailableForApplications_IdAndVisible(app, "Y");
|
|
||||||
|
User user = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
|
||||||
|
if(user.getRole().name().equals("ADMIN")){
|
||||||
|
scenAccept.add("DRAFT");
|
||||||
|
}
|
||||||
|
lstScenarios = scenarioRepo.findByAvailableForApplications_IdAndVisibleIn(app, scenAccept);
|
||||||
|
//
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
logger.error("Exception ScenarioRepository:", e.getMessage());
|
logger.error("Exception ScenarioRepository:", e.getMessage());
|
||||||
}
|
}
|
||||||
@@ -55,8 +73,16 @@ public class ScenarioService {
|
|||||||
public List<Scenario> getListScenariosCross(){
|
public List<Scenario> getListScenariosCross(){
|
||||||
logger.info("getListProjectByUser function:");
|
logger.info("getListProjectByUser function:");
|
||||||
List<Scenario> lstScenarios = null;
|
List<Scenario> lstScenarios = null;
|
||||||
|
|
||||||
|
List<String> scenAccept = new ArrayList<String>();
|
||||||
|
scenAccept.add("Y");
|
||||||
try{
|
try{
|
||||||
lstScenarios = scenarioRepo.findByAvailableForProjectsIsNullAndAvailableForApplicationsIsNullAndVisible("Y");
|
|
||||||
|
User user = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
|
||||||
|
if(user.getRole().name().equals("ADMIN")){
|
||||||
|
scenAccept.add("DRAFT");
|
||||||
|
}
|
||||||
|
lstScenarios = scenarioRepo.findByAvailableForProjectsIsNullAndAvailableForApplicationsIsNullAndVisibleIn(scenAccept);
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
logger.error("Exception ScenarioRepository:", e.getMessage());
|
logger.error("Exception ScenarioRepository:", e.getMessage());
|
||||||
}
|
}
|
||||||
@@ -72,10 +98,16 @@ public class ScenarioService {
|
|||||||
Scenario scenarioDefault = new Scenario();
|
Scenario scenarioDefault = new Scenario();
|
||||||
scenarioDefault.setName("Default");
|
scenarioDefault.setName("Default");
|
||||||
scenarioDefault.setId("");
|
scenarioDefault.setId("");
|
||||||
|
|
||||||
|
List<String> scenAccept = new ArrayList<String>();
|
||||||
|
scenAccept.add("Y");
|
||||||
try{
|
try{
|
||||||
|
|
||||||
User user = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
|
User user = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
|
||||||
lstScenarios = scenarioRepo.findByVisibleAndCategoryAndProjectOrApplication(user.getSelectedProject().getId(), user.getSelectedApplication().getId());
|
if(user.getRole().name().equals("ADMIN")){
|
||||||
|
scenAccept.add("DRAFT");
|
||||||
|
}
|
||||||
|
lstScenarios = scenarioRepo.findByVisibleInAndCategoryAndProjectOrApplication(user.getSelectedProject().getId(), user.getSelectedApplication().getId(), scenAccept);
|
||||||
lstScenarios.add(scenarioDefault);
|
lstScenarios.add(scenarioDefault);
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
logger.error("Exception ScenarioRepository:", e.getMessage());
|
logger.error("Exception ScenarioRepository:", e.getMessage());
|
||||||
|
|||||||
@@ -104,217 +104,218 @@ public class ExternalCodeGenieSolver extends StepSolver {
|
|||||||
@Override
|
@Override
|
||||||
public ScenarioExecution solveStep() throws Exception {
|
public ScenarioExecution solveStep() throws Exception {
|
||||||
|
|
||||||
try {
|
System.out.println("Solving step: " + this.step.getName());
|
||||||
|
|
||||||
System.out.println("Solving step: " + this.step.getName());
|
this.scenarioExecution.setCurrentStepId(this.step.getStepId());
|
||||||
|
|
||||||
this.scenarioExecution.setCurrentStepId(this.step.getStepId());
|
loadParameters();
|
||||||
|
|
||||||
loadParameters();
|
// token
|
||||||
|
HttpHeaders headersToken = new HttpHeaders();
|
||||||
|
headersToken.setContentType(MediaType.APPLICATION_JSON);
|
||||||
|
// headers.set("Authorization", "Bearer ");
|
||||||
|
String auth = codegenie_username + ":" + codegenie_password;
|
||||||
|
String encodedAuth = java.util.Base64.getEncoder().encodeToString(auth.getBytes());
|
||||||
|
headersToken.set("Authorization", "Basic " + encodedAuth);
|
||||||
|
|
||||||
// token
|
HttpEntity<Void> entityToken = new HttpEntity<>(headersToken);
|
||||||
HttpHeaders headersToken = new HttpHeaders();
|
|
||||||
headersToken.setContentType(MediaType.APPLICATION_JSON);
|
|
||||||
// headers.set("Authorization", "Bearer ");
|
|
||||||
String auth = codegenie_username + ":" + codegenie_password;
|
|
||||||
String encodedAuth = java.util.Base64.getEncoder().encodeToString(auth.getBytes());
|
|
||||||
headersToken.set("Authorization", "Basic " + encodedAuth);
|
|
||||||
|
|
||||||
HttpEntity<Void> entityToken = new HttpEntity<>(headersToken);
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
|
||||||
RestTemplate restTemplate = new RestTemplate();
|
ResponseEntity<String> response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/token",
|
||||||
|
HttpMethod.POST,
|
||||||
|
entityToken,
|
||||||
|
String.class);
|
||||||
|
JSONObject jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
ResponseEntity<String> response = restTemplate.exchange(
|
List<String> output_type = new ArrayList<String>();
|
||||||
|
output_type.add(this.codegenie_output_type);
|
||||||
|
|
||||||
|
JSONObject requestBody = new JSONObject();
|
||||||
|
requestBody.put("execution_id", this.scenarioExecution.getId());
|
||||||
|
requestBody.put("docs_zip_file", this.codegenie_input);
|
||||||
|
requestBody.put("application", this.codegenie_application);
|
||||||
|
requestBody.put("project", this.codegenie_project);
|
||||||
|
requestBody.put("output_type", output_type);
|
||||||
|
requestBody.put("output_variable", this.codegenie_output_variable);
|
||||||
|
requestBody.put("model_provider", this.codegenie_model_provider);
|
||||||
|
|
||||||
|
HttpHeaders headers = new HttpHeaders();
|
||||||
|
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||||
|
headers.set("Authorization",
|
||||||
|
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
||||||
|
HttpEntity<String> request = new HttpEntity<>(requestBody.toString(), headers);
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/execute",
|
||||||
|
HttpMethod.POST,
|
||||||
|
request,
|
||||||
|
String.class);
|
||||||
|
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
|
/*
|
||||||
|
* response = restTemplate.exchange(
|
||||||
|
* //this.codegenie_base_url + "/execution_status/" +
|
||||||
|
* this.scenarioExecution.getId(),
|
||||||
|
* this.codegenie_base_url + "/execution_status/679752f85189eb5621b48e17",
|
||||||
|
* HttpMethod.GET,
|
||||||
|
* request,
|
||||||
|
* String.class);
|
||||||
|
* jsonResponse = new JSONObject(response.getBody());
|
||||||
|
*/
|
||||||
|
|
||||||
|
int maxTries = 500;
|
||||||
|
// Pool the status GET api until it return the SUCCESS or FAILED message
|
||||||
|
while (!jsonResponse.get("status").equals("DONE") && !jsonResponse.get("status").equals("ERROR")
|
||||||
|
&& maxTries > 0) {
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
this.codegenie_base_url + "/token",
|
this.codegenie_base_url + "/token",
|
||||||
HttpMethod.POST,
|
HttpMethod.POST,
|
||||||
entityToken,
|
entityToken,
|
||||||
String.class);
|
String.class);
|
||||||
JSONObject jsonResponse = new JSONObject(response.getBody());
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
List<String> output_type = new ArrayList<String>();
|
|
||||||
output_type.add(this.codegenie_output_type);
|
|
||||||
|
|
||||||
JSONObject requestBody = new JSONObject();
|
|
||||||
requestBody.put("execution_id", this.scenarioExecution.getId());
|
|
||||||
requestBody.put("docs_zip_file", this.codegenie_input);
|
|
||||||
requestBody.put("application", this.codegenie_application);
|
|
||||||
requestBody.put("project", this.codegenie_project);
|
|
||||||
requestBody.put("output_type", output_type);
|
|
||||||
requestBody.put("output_variable", this.codegenie_output_variable);
|
|
||||||
requestBody.put("model_provider", this.codegenie_model_provider);
|
|
||||||
|
|
||||||
HttpHeaders headers = new HttpHeaders();
|
|
||||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
|
||||||
headers.set("Authorization",
|
headers.set("Authorization",
|
||||||
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
||||||
HttpEntity<String> request = new HttpEntity<>(requestBody.toString(), headers);
|
request = new HttpEntity<>(requestBody.toString(), headers);
|
||||||
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/execute",
|
|
||||||
HttpMethod.POST,
|
|
||||||
request,
|
|
||||||
String.class);
|
|
||||||
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
/* response = restTemplate.exchange(
|
|
||||||
//this.codegenie_base_url + "/execution_status/" + this.scenarioExecution.getId(),
|
|
||||||
this.codegenie_base_url + "/execution_status/679752f85189eb5621b48e17",
|
|
||||||
HttpMethod.GET,
|
|
||||||
request,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());*/
|
|
||||||
|
|
||||||
int maxTries = 500;
|
|
||||||
// Pool the status GET api until it return the SUCCESS or FAILED message
|
|
||||||
while (!jsonResponse.get("status").equals("DONE") && !jsonResponse.get("status").equals("ERROR")
|
|
||||||
&& maxTries > 0) {
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/token",
|
|
||||||
HttpMethod.POST,
|
|
||||||
entityToken,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
headers.set("Authorization",
|
|
||||||
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
|
||||||
request = new HttpEntity<>(requestBody.toString(), headers);
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/execution_status/" + this.scenarioExecution.getId(),
|
|
||||||
//this.codegenie_base_url + "/execution_status/679752f85189eb5621b48e17",
|
|
||||||
HttpMethod.GET,
|
|
||||||
request,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
try {
|
|
||||||
Thread.sleep(120000);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
Thread.currentThread().interrupt();
|
|
||||||
logger.error("Thread was interrupted", e);
|
|
||||||
}
|
|
||||||
logger.info("Check status => Remaining tryes :" + maxTries);
|
|
||||||
logger.info("Percent: " + jsonResponse.get("percent"));
|
|
||||||
maxTries--;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("Stop pooling codegenies pod. Latest status = " + jsonResponse.get("status"));
|
|
||||||
|
|
||||||
if (jsonResponse.get("status").equals("DONE")) {
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/token",
|
|
||||||
HttpMethod.POST,
|
|
||||||
entityToken,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
headers.set("Authorization",
|
|
||||||
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
|
||||||
request = new HttpEntity<>(requestBody.toString(), headers);
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/execution_result/" +this.scenarioExecution.getId(),
|
|
||||||
//this.codegenie_base_url + "/execution_result/679752f85189eb5621b48e17",
|
|
||||||
HttpMethod.GET,
|
|
||||||
request,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
if (jsonResponse.get("status").equals("DONE")) {
|
|
||||||
|
|
||||||
// Accedi all'oggetto "outputs"
|
|
||||||
JSONObject outputs = jsonResponse.getJSONObject("outputs");
|
|
||||||
|
|
||||||
// Ottieni il valore del campo "json"
|
|
||||||
String jsonPath = "";
|
|
||||||
|
|
||||||
if (this.codegenie_output_type.equals("FILE")) {
|
|
||||||
|
|
||||||
jsonPath = outputs.getString("file");
|
|
||||||
|
|
||||||
} else if (this.codegenie_output_type.equals("MARKDOWN")) {
|
|
||||||
jsonPath = outputs.getString("markdown");
|
|
||||||
// this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
|
||||||
// jsonResponse.get("OUTPUT").toString());
|
|
||||||
} else if (this.codegenie_output_type.equals("JSON")) {
|
|
||||||
jsonPath = outputs.getString("json");
|
|
||||||
// this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
|
||||||
// jsonResponse.get("OUTPUT").toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/token",
|
|
||||||
HttpMethod.POST,
|
|
||||||
entityToken,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
headers.set("Authorization",
|
|
||||||
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
|
||||||
request = new HttpEntity<>(requestBody.toString(), headers);
|
|
||||||
|
|
||||||
ResponseEntity<byte[]> responseFile = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/download/" + jsonPath,
|
|
||||||
HttpMethod.GET,
|
|
||||||
request,
|
|
||||||
byte[].class);
|
|
||||||
|
|
||||||
// Salva i bytes in un file
|
|
||||||
byte[] fileBytes = responseFile.getBody();
|
|
||||||
String fileBase64 = Base64.getEncoder().encodeToString(fileBytes);
|
|
||||||
|
|
||||||
this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
|
||||||
fileBase64);
|
|
||||||
this.scenarioExecution.setNextStepId(this.step.getNextStepId());
|
|
||||||
this.scenarioExecution.getExecSharedMap().put("status",
|
|
||||||
"DONE");
|
|
||||||
} else {
|
|
||||||
throw new Exception(
|
|
||||||
"codegenie execution failed with status: " + jsonResponse.get("status").toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
logger.error("ERROR on pooling codegenies");
|
|
||||||
|
|
||||||
throw new Exception("codegenie execution failed with status: " + jsonResponse.get("status").toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
response = restTemplate.exchange(
|
||||||
this.codegenie_base_url + "/token",
|
this.codegenie_base_url + "/execution_status/" + this.scenarioExecution.getId(),
|
||||||
HttpMethod.POST,
|
// this.codegenie_base_url + "/execution_status/679752f85189eb5621b48e17",
|
||||||
entityToken,
|
|
||||||
String.class);
|
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
|
||||||
|
|
||||||
headers.set("Authorization",
|
|
||||||
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
|
||||||
request = new HttpEntity<>(requestBody.toString(), headers);
|
|
||||||
|
|
||||||
response = restTemplate.exchange(
|
|
||||||
this.codegenie_base_url + "/execution_consumption/" + this.scenarioExecution.getId(),
|
|
||||||
//this.codegenie_base_url + "/execution_consumption/679752f85189eb5621b48e17",
|
|
||||||
HttpMethod.GET,
|
HttpMethod.GET,
|
||||||
request,
|
request,
|
||||||
String.class);
|
String.class);
|
||||||
jsonResponse = new JSONObject(response.getBody());
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
this.scenarioExecution.getExecSharedMap().put("input_token", jsonResponse.get("input_token").toString());
|
|
||||||
this.scenarioExecution.getExecSharedMap().put("output_token", jsonResponse.get("output_token").toString());
|
|
||||||
this.scenarioExecution.getExecSharedMap().put("total_token", jsonResponse.get("total_token").toString());
|
|
||||||
this.scenarioExecution.getExecSharedMap().put("used_model", jsonResponse.get("used_model").toString());
|
|
||||||
|
|
||||||
// scenarioExecutionRepo.save(this.scenarioExecution);
|
try {
|
||||||
|
Thread.sleep(120000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
logger.error("Thread was interrupted", e);
|
||||||
|
}
|
||||||
|
logger.info("Check status => Remaining tryes :" + maxTries);
|
||||||
|
logger.info("Percent: " + jsonResponse.get("percent"));
|
||||||
|
maxTries--;
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("Error in codegenie execution: " + e.getMessage(), e);
|
|
||||||
this.scenarioExecution.getExecSharedMap().put("status",
|
|
||||||
"ERROR");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.info("Stop pooling codegenies pod. Latest status = " + jsonResponse.get("status"));
|
||||||
|
|
||||||
|
if (jsonResponse.get("status").equals("DONE")) {
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/token",
|
||||||
|
HttpMethod.POST,
|
||||||
|
entityToken,
|
||||||
|
String.class);
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
|
headers.set("Authorization",
|
||||||
|
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
||||||
|
request = new HttpEntity<>(requestBody.toString(), headers);
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/execution_result/" + this.scenarioExecution.getId(),
|
||||||
|
// this.codegenie_base_url + "/execution_result/679752f85189eb5621b48e17",
|
||||||
|
HttpMethod.GET,
|
||||||
|
request,
|
||||||
|
String.class);
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
|
if (jsonResponse.get("status").equals("DONE")) {
|
||||||
|
|
||||||
|
// Accedi all'oggetto "outputs"
|
||||||
|
JSONObject outputs = jsonResponse.getJSONObject("outputs");
|
||||||
|
|
||||||
|
// Ottieni il valore del campo "json"
|
||||||
|
String jsonPath = "";
|
||||||
|
|
||||||
|
if (this.codegenie_output_type.equals("FILE")) {
|
||||||
|
|
||||||
|
jsonPath = outputs.getString("file");
|
||||||
|
|
||||||
|
} else if (this.codegenie_output_type.equals("MARKDOWN")) {
|
||||||
|
jsonPath = outputs.getString("markdown");
|
||||||
|
// this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
||||||
|
// jsonResponse.get("OUTPUT").toString());
|
||||||
|
} else if (this.codegenie_output_type.equals("JSON")) {
|
||||||
|
jsonPath = outputs.getString("json");
|
||||||
|
// this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
||||||
|
// jsonResponse.get("OUTPUT").toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/token",
|
||||||
|
HttpMethod.POST,
|
||||||
|
entityToken,
|
||||||
|
String.class);
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
|
headers.set("Authorization",
|
||||||
|
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
||||||
|
request = new HttpEntity<>(requestBody.toString(), headers);
|
||||||
|
|
||||||
|
ResponseEntity<byte[]> responseFile = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/download/" + jsonPath,
|
||||||
|
HttpMethod.GET,
|
||||||
|
request,
|
||||||
|
byte[].class);
|
||||||
|
|
||||||
|
// Salva i bytes in un file
|
||||||
|
byte[] fileBytes = responseFile.getBody();
|
||||||
|
String fileBase64 = Base64.getEncoder().encodeToString(fileBytes);
|
||||||
|
|
||||||
|
this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
||||||
|
fileBase64);
|
||||||
|
this.scenarioExecution.setNextStepId(this.step.getNextStepId());
|
||||||
|
this.scenarioExecution.getExecSharedMap().put("status",
|
||||||
|
"DONE");
|
||||||
|
} else {
|
||||||
|
|
||||||
|
this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
||||||
|
"Error while generating file");
|
||||||
|
throw new Exception(
|
||||||
|
"codegenie execution failed with status: " + jsonResponse.get("status").toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
logger.error("ERROR on pooling codegenies");
|
||||||
|
this.scenarioExecution.getExecSharedMap().put(this.codegenie_output_variable,
|
||||||
|
"Error while generating file");
|
||||||
|
|
||||||
|
throw new Exception("codegenie execution failed with status: " + jsonResponse.get("status").toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/token",
|
||||||
|
HttpMethod.POST,
|
||||||
|
entityToken,
|
||||||
|
String.class);
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
|
||||||
|
headers.set("Authorization",
|
||||||
|
jsonResponse.get("token_type").toString() + " " + jsonResponse.get("access_token").toString());
|
||||||
|
request = new HttpEntity<>(requestBody.toString(), headers);
|
||||||
|
|
||||||
|
response = restTemplate.exchange(
|
||||||
|
this.codegenie_base_url + "/execution_consumption/" + this.scenarioExecution.getId(),
|
||||||
|
// this.codegenie_base_url + "/execution_consumption/679752f85189eb5621b48e17",
|
||||||
|
HttpMethod.GET,
|
||||||
|
request,
|
||||||
|
String.class);
|
||||||
|
jsonResponse = new JSONObject(response.getBody());
|
||||||
|
this.scenarioExecution.getExecSharedMap().put("input_token", jsonResponse.get("input_token").toString());
|
||||||
|
this.scenarioExecution.getExecSharedMap().put("output_token", jsonResponse.get("output_token").toString());
|
||||||
|
this.scenarioExecution.getExecSharedMap().put("total_token", jsonResponse.get("total_token").toString());
|
||||||
|
this.scenarioExecution.getExecSharedMap().put("used_model", jsonResponse.get("used_model").toString());
|
||||||
|
|
||||||
|
// scenarioExecutionRepo.save(this.scenarioExecution);
|
||||||
|
|
||||||
|
this.scenarioExecution.setNextStepId(this.step.getNextStepId());
|
||||||
|
|
||||||
return this.scenarioExecution;
|
return this.scenarioExecution;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -36,20 +36,18 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
private String qai_system_prompt_template_chunk;
|
private String qai_system_prompt_template_chunk;
|
||||||
private String qai_path_file;
|
private String qai_path_file;
|
||||||
private String qai_output_variable;
|
private String qai_output_variable;
|
||||||
private int chunk_size_token;
|
private int max_chunk_size_token;
|
||||||
private String encoding_name;
|
private String encoding_name;
|
||||||
private double charMax;
|
|
||||||
private int tokenCount;
|
private int tokenCount;
|
||||||
private Optional<Encoding> encoding;
|
private Optional<Encoding> encoding;
|
||||||
private int percent_summarize;
|
private Double percent_chunk_length;
|
||||||
private Integer maxTokenChunk;
|
|
||||||
private String qai_custom_memory_id;
|
private String qai_custom_memory_id;
|
||||||
private Integer max_output_token;
|
private Integer max_output_token;
|
||||||
private String qai_system_prompt_template_minimum;
|
private Integer max_input_token_not_to_be_chunked;
|
||||||
private String qai_system_prompt_template_formatter;
|
private String qai_system_prompt_template_formatter;
|
||||||
private boolean isChunked = false;
|
private boolean isChunked = false;
|
||||||
private Double chunk_size_token_calc;
|
private Double chunk_size_token_calc;
|
||||||
private Double perc = 0.2;
|
|
||||||
// private boolean qai_load_graph_schema=false;
|
// private boolean qai_load_graph_schema=false;
|
||||||
|
|
||||||
Logger logger = (Logger) LoggerFactory.getLogger(SummarizeDocSolver.class);
|
Logger logger = (Logger) LoggerFactory.getLogger(SummarizeDocSolver.class);
|
||||||
@@ -68,16 +66,15 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
this.qai_output_variable = (String) this.step.getAttributes().get("qai_output_variable");
|
this.qai_output_variable = (String) this.step.getAttributes().get("qai_output_variable");
|
||||||
this.encoding_name = (String) this.step.getAttributes().get("encoding_name");
|
this.encoding_name = (String) this.step.getAttributes().get("encoding_name");
|
||||||
|
|
||||||
this.chunk_size_token = Integer.parseInt((String) this.step.getAttributes().get("chunk_size_token"));
|
this.max_chunk_size_token = Integer.parseInt((String) this.step.getAttributes().get("max_chunk_size_token"));
|
||||||
|
|
||||||
this.percent_summarize = Integer.parseInt((String) this.step.getAttributes().get("percent_summarize"));
|
this.percent_chunk_length = (Double) this.step.getAttributes().get("percent_chunk_length");
|
||||||
|
|
||||||
this.qai_custom_memory_id = (String) this.step.getAttributes().get("qai_custom_memory_id");
|
this.qai_custom_memory_id = (String) this.step.getAttributes().get("qai_custom_memory_id");
|
||||||
|
|
||||||
this.max_output_token = Integer.parseInt((String) this.step.getAttributes().get("max_output_token"));
|
this.max_output_token = Integer.parseInt((String) this.step.getAttributes().get("max_output_token"));
|
||||||
|
|
||||||
this.qai_system_prompt_template_minimum = attributeParser
|
this.max_input_token_not_to_be_chunked = Integer.parseInt((String) this.step.getAttributes().get("max_input_token_not_to_be_chunked"));
|
||||||
.parse((String) this.step.getAttributes().get("qai_system_prompt_template_minimum"));
|
|
||||||
|
|
||||||
this.qai_system_prompt_template_formatter = attributeParser
|
this.qai_system_prompt_template_formatter = attributeParser
|
||||||
.parse((String) this.step.getAttributes().get("qai_system_prompt_template_formatter"));
|
.parse((String) this.step.getAttributes().get("qai_system_prompt_template_formatter"));
|
||||||
@@ -102,38 +99,21 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
EncodingRegistry registry = Encodings.newDefaultEncodingRegistry();
|
EncodingRegistry registry = Encodings.newDefaultEncodingRegistry();
|
||||||
encoding = registry.getEncoding(this.encoding_name);
|
encoding = registry.getEncoding(this.encoding_name);
|
||||||
|
|
||||||
// Conta i token
|
|
||||||
// int tokenCount = encoding.get().encode(text).size();
|
|
||||||
tokenCount = encoding.get().countTokens(text);
|
tokenCount = encoding.get().countTokens(text);
|
||||||
logger.info("token count input: " + tokenCount);
|
logger.info("token count input: " + tokenCount);
|
||||||
int charCount = text.length();
|
|
||||||
|
|
||||||
// Stima media caratteri per token
|
// Calcolo della dimensione del chunk in token
|
||||||
// double charPerToken = (double) charCount / tokenCount;
|
chunk_size_token_calc = (Double)((double)tokenCount * percent_chunk_length);
|
||||||
|
if(chunk_size_token_calc>max_chunk_size_token){
|
||||||
// Double output_charD = (double) charCount * ((double) this.percent_summarize / 100.0);
|
chunk_size_token_calc = (double)max_chunk_size_token;
|
||||||
// Integer output_char = output_charD.intValue();
|
|
||||||
Double min_output_tokenD = (double) tokenCount * ((double) this.percent_summarize / 100.0);
|
|
||||||
Integer min_output_token = min_output_tokenD.intValue();
|
|
||||||
String content = new String("");
|
|
||||||
content = this.qai_system_prompt_template.replace("max_number_token",
|
|
||||||
max_output_token.toString());
|
|
||||||
|
|
||||||
if (min_output_token < max_output_token) {
|
|
||||||
content += this.qai_system_prompt_template_minimum.replace(
|
|
||||||
"min_number_token",
|
|
||||||
min_output_token.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
chunk_size_token_calc = (Double)((double)tokenCount * perc);
|
|
||||||
if(chunk_size_token_calc>chunk_size_token){
|
|
||||||
chunk_size_token_calc = (double)chunk_size_token;
|
|
||||||
|
|
||||||
}
|
}
|
||||||
// **Fase di Summarization**
|
// **Fase di Summarization**
|
||||||
String summarizedText = summarize(text); // 🔹 Applica la funzione di riassunto
|
String summarizedText = text;
|
||||||
// String template = this.qai_system_prompt_template+" The output length should
|
|
||||||
// be of " + output_char + " characters";
|
if(tokenCount > max_input_token_not_to_be_chunked){
|
||||||
|
summarizedText = summarize(text);
|
||||||
|
}
|
||||||
|
|
||||||
// Creazione dei messaggi per il modello AI
|
// Creazione dei messaggi per il modello AI
|
||||||
Message userMessage = new UserMessage(summarizedText);
|
Message userMessage = new UserMessage(summarizedText);
|
||||||
@@ -146,8 +126,8 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
logger.info("template formatter: " + this.qai_system_prompt_template_formatter);
|
logger.info("template formatter: " + this.qai_system_prompt_template_formatter);
|
||||||
}else{
|
}else{
|
||||||
//here
|
//here
|
||||||
systemMessage = new SystemMessage(content);
|
systemMessage = new SystemMessage(this.qai_system_prompt_template);
|
||||||
logger.info("template: " + content);
|
logger.info("template: " + this.qai_system_prompt_template);
|
||||||
}
|
}
|
||||||
|
|
||||||
CallResponseSpec resp = chatClient.prompt()
|
CallResponseSpec resp = chatClient.prompt()
|
||||||
@@ -185,7 +165,6 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String summarize(String text) {
|
private String summarize(String text) {
|
||||||
// Se il testo è già corto, non riassumere
|
|
||||||
logger.info("length: " + text.length());
|
logger.info("length: " + text.length());
|
||||||
Double chunk_size_text;
|
Double chunk_size_text;
|
||||||
|
|
||||||
@@ -205,10 +184,6 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
List<String> chunks = chunkText(text, chunk_size_text.intValue());
|
List<String> chunks = chunkText(text, chunk_size_text.intValue());
|
||||||
List<String> summarizedChunks = new ArrayList<>();
|
List<String> summarizedChunks = new ArrayList<>();
|
||||||
|
|
||||||
//Double maxTokenChunkD = Math.ceil(chunk_size_token / ratio);
|
|
||||||
Double maxTokenChunkD = Math.ceil(chunk_size_token_calc / ratio);
|
|
||||||
maxTokenChunk = maxTokenChunkD.intValue();
|
|
||||||
|
|
||||||
// Riassumere ogni chunk singolarmente
|
// Riassumere ogni chunk singolarmente
|
||||||
for (String chunk : chunks) {
|
for (String chunk : chunks) {
|
||||||
logger.info("chunk length: " + chunk.length());
|
logger.info("chunk length: " + chunk.length());
|
||||||
@@ -240,24 +215,12 @@ public class SummarizeDocSolver extends StepSolver {
|
|||||||
|
|
||||||
// Metodo di riassunto per singolo chunk
|
// Metodo di riassunto per singolo chunk
|
||||||
private String summarizeChunk(String chunk) {
|
private String summarizeChunk(String chunk) {
|
||||||
String content = new String("");
|
|
||||||
|
|
||||||
/* if (maxTokenChunk < max_output_token) {
|
|
||||||
content = this.qai_system_prompt_template_chunk.replace("max_number_token",
|
|
||||||
maxTokenChunk.toString());
|
|
||||||
}else{
|
|
||||||
content = this.qai_system_prompt_template_chunk.replace("max_number_token",
|
|
||||||
max_output_token.toString());
|
|
||||||
}*/
|
|
||||||
|
|
||||||
content = this.qai_system_prompt_template_chunk.replace("max_number_token",
|
|
||||||
maxTokenChunk.toString());
|
|
||||||
|
|
||||||
Message chunkMessage = new UserMessage(chunk);
|
Message chunkMessage = new UserMessage(chunk);
|
||||||
Message systemMessage = new SystemMessage(
|
Message systemMessage = new SystemMessage(
|
||||||
content);
|
this.qai_system_prompt_template_chunk);
|
||||||
|
|
||||||
logger.info("template chunk: " + content);
|
logger.info("template chunk: " + this.qai_system_prompt_template_chunk);
|
||||||
|
|
||||||
CallResponseSpec resp = chatClient.prompt()
|
CallResponseSpec resp = chatClient.prompt()
|
||||||
.messages(chunkMessage, systemMessage)
|
.messages(chunkMessage, systemMessage)
|
||||||
|
|||||||
Reference in New Issue
Block a user