From dbeb7e4a64f433b5c092745118b01893769f87b0 Mon Sep 17 00:00:00 2001 From: "andrea.terzani" Date: Thu, 21 Nov 2024 11:28:49 +0100 Subject: [PATCH 1/8] Aggiunta del supporto per l'agente esterno e modifica dei metodi di esecuzione --- .../services/ApplicationBrowserService.java | 4 +- .../services/ScenarioExecutionService.java | 5 +- .../stepSolvers/ExternalAgentSolver.java | 141 ++++++++++++++++++ .../hermione/stepSolvers/StepSolver.java | 2 +- 4 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java diff --git a/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java b/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java index 23d7345..8124781 100644 --- a/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java +++ b/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java @@ -39,7 +39,7 @@ public class ApplicationBrowserService { classInfo.setCommitSha(kSGitInfo.getCommitId()); classInfo.setRepositoryEntityId(kSGitInfo.getId()); - result = javaREModule.revSingleClass(classInfo); + result = null;//javaREModule.revSingleClass(classInfo); } @@ -56,7 +56,7 @@ public class ApplicationBrowserService { try { // Chiama il metodo getProcessStatus e memorizza il risultato - responseEntity = javaREModule.getProcessStatus(processId); + responseEntity = null;//javaREModule.getProcessStatus(processId); } catch (Exception e) { // In caso di eccezione logger.error("Exception in getProgressRev: {}", e.getMessage()); diff --git a/src/main/java/com/olympus/hermione/services/ScenarioExecutionService.java b/src/main/java/com/olympus/hermione/services/ScenarioExecutionService.java index 45ac5ca..69153e2 100644 --- a/src/main/java/com/olympus/hermione/services/ScenarioExecutionService.java +++ b/src/main/java/com/olympus/hermione/services/ScenarioExecutionService.java @@ -47,6 +47,7 @@ import org.slf4j.Logger; import com.azure.ai.openai.OpenAIClient; import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; +import com.olympus.hermione.stepSolvers.ExternalAgentSolver; @@ -231,7 +232,9 @@ public class ScenarioExecutionService { case "RAG_SOURCE_CODE": solver = new SourceCodeRagSolver(); break; - + case "EXTERNAL_AGENT": + solver = new ExternalAgentSolver(); + break; default: break; } diff --git a/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java new file mode 100644 index 0000000..d40b9cb --- /dev/null +++ b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java @@ -0,0 +1,141 @@ +package com.olympus.hermione.stepSolvers; +import org.json.JSONObject; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +import com.olympus.hermione.models.ScenarioExecution; +import com.olympus.hermione.utility.AttributeParser; + +import ch.qos.logback.classic.Logger; + +public class ExternalAgentSolver extends StepSolver { + + private String agent_input; + private String agent_base_url; + private String agent_output_variable; + private String agent_application; + private String agent_project; + + Logger logger = (Logger) LoggerFactory.getLogger(BasicQueryRagSolver.class); + + private void loadParameters(){ + logger.info("Loading parameters"); + + if(this.step.getAttributes().get("agent_input")!=null ){ + this.agent_input = (String) this.step.getAttributes().get("agent_input"); + logger.info("agent_input: " + this.agent_input); + } + + if(this.step.getAttributes().get("agent_application")!=null ){ + this.agent_input = (String) this.step.getAttributes().get("agent_application"); + logger.info("agent_application: " + this.agent_application); + } + if(this.step.getAttributes().get("agent_project")!=null ){ + this.agent_project = (String) this.step.getAttributes().get("agent_project"); + logger.info("agent_project: " + this.agent_project); + } + + if(this.step.getAttributes().get("agent_base_url")!=null ){ + this.agent_base_url = (String) this.step.getAttributes().get("agent_base_url"); + logger.info("agent_base_url: " + this.agent_base_url); + } + + if(this.step.getAttributes().get("agent_output_variable")!=null ){ + this.agent_output_variable = (String) this.step.getAttributes().get("agent_output_variable"); + logger.info("agent_output_variable: " + this.agent_output_variable); + } + + AttributeParser attributeParser = new AttributeParser(this.scenarioExecution); + + this.agent_input = attributeParser.parse((String) this.step.getAttributes().get("agent_input")); + this.agent_application = attributeParser.parse((String) this.step.getAttributes().get("agent_application")); + this.agent_project = attributeParser.parse((String) this.step.getAttributes().get("agent_project")); + + + } + + @Override + public ScenarioExecution solveStep() throws Exception { + + System.out.println("Solving step: " + this.step.getName()); + + this.scenarioExecution.setCurrentStepId(this.step.getStepId()); + + loadParameters(); + + + RestTemplate restTemplate = new RestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + JSONObject requestBody = new JSONObject(); + requestBody.put("executionId", this.scenarioExecution.getId()); + requestBody.put("user_input", this.agent_input); + requestBody.put("application", this.agent_application); + requestBody.put("project", this.agent_project); + + HttpEntity request = new HttpEntity<>(requestBody.toString(), headers); + + ResponseEntity response = restTemplate.exchange( + this.agent_base_url+"/execute", + HttpMethod.POST, + request, + String.class + ); + + JSONObject jsonResponse = new JSONObject(response.getBody()); + + if (!jsonResponse.get("status").equals("STARTED")) { + throw new Exception("Agent execution failed with status: " + jsonResponse.get("status")); + } + + + int maxTries = 100; + // Pool the status GET api until it return the SUCCESS or FAILED message + while(!jsonResponse.get("status").equals("COMPLETED") && !jsonResponse.get("status").equals("FAILED") && maxTries > 0){ + response = restTemplate.exchange( + this.agent_base_url + "/status/" + this.scenarioExecution.getId(), + HttpMethod.GET, + request, + String.class + ); + jsonResponse = new JSONObject(response.getBody()); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Thread was interrupted", e); + } + + maxTries--; + + } + + if(jsonResponse.get("status").equals("COMPLETED")){ + response = restTemplate.exchange( + this.agent_base_url + "/execution_result/" + this.scenarioExecution.getId(), + HttpMethod.GET, + request, + String.class + ); + jsonResponse = new JSONObject(response.getBody()); + + this.scenarioExecution.getExecSharedMap().put(this.agent_output_variable, jsonResponse.toString()); + this.scenarioExecution.setNextStepId(this.step.getNextStepId()); + + }else{ + throw new Exception("Agent execution failed with status: " + jsonResponse.get("status")); + } + + + + return this.scenarioExecution; + } + +} diff --git a/src/main/java/com/olympus/hermione/stepSolvers/StepSolver.java b/src/main/java/com/olympus/hermione/stepSolvers/StepSolver.java index e18f343..cdf498e 100644 --- a/src/main/java/com/olympus/hermione/stepSolvers/StepSolver.java +++ b/src/main/java/com/olympus/hermione/stepSolvers/StepSolver.java @@ -27,7 +27,7 @@ public class StepSolver { private Logger logger = LoggerFactory.getLogger(StepSolver.class); - public ScenarioExecution solveStep(){ + public ScenarioExecution solveStep() throws Exception { logger.info("Solving step: " + this.step.getName()); this.scenarioExecution.setCurrentStepId(this.step.getStepId()); this.scenarioExecution.setNextStepId(this.step.getNextStepId()); From 9efedc4ebecfb4a151189352d84b7abb6c59234f Mon Sep 17 00:00:00 2001 From: Andrea Terzani Date: Thu, 21 Nov 2024 17:07:28 +0000 Subject: [PATCH 2/8] Updated ApplicationBrowserService.java --- .../olympus/hermione/services/ApplicationBrowserService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java b/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java index 8124781..23d7345 100644 --- a/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java +++ b/src/main/java/com/olympus/hermione/services/ApplicationBrowserService.java @@ -39,7 +39,7 @@ public class ApplicationBrowserService { classInfo.setCommitSha(kSGitInfo.getCommitId()); classInfo.setRepositoryEntityId(kSGitInfo.getId()); - result = null;//javaREModule.revSingleClass(classInfo); + result = javaREModule.revSingleClass(classInfo); } @@ -56,7 +56,7 @@ public class ApplicationBrowserService { try { // Chiama il metodo getProcessStatus e memorizza il risultato - responseEntity = null;//javaREModule.getProcessStatus(processId); + responseEntity = javaREModule.getProcessStatus(processId); } catch (Exception e) { // In caso di eccezione logger.error("Exception in getProgressRev: {}", e.getMessage()); From c85e6c8e3cfbae3fa127f6dbb32316607aad6841 Mon Sep 17 00:00:00 2001 From: Andrea Terzani Date: Fri, 22 Nov 2024 16:09:43 +0000 Subject: [PATCH 3/8] Updated hermione-deployment.yaml --- manifests/hermione-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manifests/hermione-deployment.yaml b/manifests/hermione-deployment.yaml index 9ae12c5..5eab9b8 100644 --- a/manifests/hermione-deployment.yaml +++ b/manifests/hermione-deployment.yaml @@ -16,6 +16,9 @@ spec: containers: - name: hermione image: olympusreg.azurecr.io/hermione:a843dee4 + env: + name: DEFAULT_CONNECTION_IDLE_TIMEOUT + value: 300 envFrom: - configMapRef: name: olympus-db-shared-cfg From de4040177ceba9fec26ca399f9e645bc7e84f994 Mon Sep 17 00:00:00 2001 From: Andrea Terzani Date: Fri, 22 Nov 2024 16:17:16 +0000 Subject: [PATCH 4/8] Updated hermione-deployment.yaml --- manifests/hermione-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/hermione-deployment.yaml b/manifests/hermione-deployment.yaml index 5eab9b8..5a15448 100644 --- a/manifests/hermione-deployment.yaml +++ b/manifests/hermione-deployment.yaml @@ -17,8 +17,8 @@ spec: - name: hermione image: olympusreg.azurecr.io/hermione:a843dee4 env: - name: DEFAULT_CONNECTION_IDLE_TIMEOUT - value: 300 + - name: DEFAULT_CONNECTION_IDLE_TIMEOUT + value: "300" envFrom: - configMapRef: name: olympus-db-shared-cfg From 8b591dbecfa9007e7562d7886900319f2911ecdb Mon Sep 17 00:00:00 2001 From: "andrea.terzani" Date: Tue, 26 Nov 2024 12:31:49 +0100 Subject: [PATCH 5/8] =?UTF-8?q?Aggiunta=20della=20dipendenza=20spring-ai-c?= =?UTF-8?q?hroma-store=20e=20configurazione=20delle=20propriet=C3=A0=20per?= =?UTF-8?q?=20il=20client=20Chroma?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 4 ++++ src/main/resources/application.properties | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index f0432bf..ccaa89b 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,10 @@ org.springframework.ai spring-ai-azure-store + + org.springframework.ai + spring-ai-chroma-store + org.neo4j.driver neo4j-java-driver diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f443bbf..6b6bdd9 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,8 +15,8 @@ spring.ai.vectorstore.mongodb.indexName=vector_index spring.ai.vectorstore.mongodb.collection-name=vector_store spring.ai.vectorstore.mongodb.initialize-schema=false -spring.ai.vectorstore.azure.api-key=jxKqZvbMKuo1MwXs8ilEAeRDeswtoTXO1lWX600jP2AzSeDXo1nq -spring.ai.vectorstore.azure.url=https://search-olympus.search.windows.net +spring.ai.vectorstore.azure.api-key=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +spring.ai.vectorstore.azure.url=https://search-olympus.search.windows.net_old spring.ai.vectorstore.azure.initialize-schema =false @@ -58,4 +58,10 @@ eureka.instance.preferIpAddress: true hermione.fe.url = http://localhost:5173 java-parser-module.url: http://java-parser-module-service.olympus.svc.cluster.local:8080 - \ No newline at end of file +java-re-module.url: http://java-re-module-service.olympus.svc.cluster.local:8080 + +spring.ai.vectorstore.chroma.client.host=http://108.142.74.161 +spring.ai.vectorstore.chroma.client.port=8000 +spring.ai.vectorstore.chroma.client.key-token=nVYLh3eq92aJP4x08dNdWngilPG2ooj9 +spring.ai.vectorstore.chroma.initialize-schema=true +spring.ai.vectorstore.chroma.collection-name=olympus \ No newline at end of file From 0a0b08b0bbe9beeb008a5b7fe518748123504b83 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 6 Dec 2024 12:05:51 +0000 Subject: [PATCH 6/8] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 86e3662..d4e2b1a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,16 +34,17 @@ stages: - script: | echo "Building the first repository..." + pwd ls -lrth displayName: 'list files' - - task: Maven@3 + - task: Maven@4 inputs: mavenPomFile: 'olympus-common/pom.xml' goals: 'install' displayName: 'Build olympus-common' - - task: Maven@3 + - task: Maven@4 inputs: mavenPomFile: 'hermione/pom.xml' goals: 'install -DskipTests' @@ -68,8 +69,15 @@ stages: docker images displayName: 'List docker images' - - upload: hermione/manifests - artifact: manifests + - script: | + echo "Removing Docker images to clean up..." + docker rmi $(docker images -q) + displayName: 'Clean up Docker Images' + continueOnError: true + condition: always() + + #- upload: hermione/manifests + # artifact: manifests - stage: Deploy displayName: Deploy stage @@ -88,6 +96,7 @@ stages: runOnce: deploy: steps: + - checkout: self - script: | echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-8)" displayName: 'Set Short Commit SHA' @@ -97,18 +106,12 @@ stages: echo "Short Commit SHA: $(shortCommitSha)" displayName: 'Echo short commit SHA' - - task: KubernetesManifest@0 + - task: KubernetesManifest@1 displayName: Deploy to Kubernetes cluster inputs: action: deploy + namespace: olympus manifests: | $(Pipeline.Workspace)/manifests/hermione-deployment.yaml containers: | - $(azurecontainerRegistry)/$(imageName):$(shortCommitSha) - - - script: | - echo "Removing Docker images to clean up..." - docker rmi $(docker images -q) - displayName: 'Clean up Docker Images' - continueOnError: true - condition: always() \ No newline at end of file + $(azurecontainerRegistry)/$(imageName):$(shortCommitSha) \ No newline at end of file From f482712cd2962a58c4823919363921d951900f70 Mon Sep 17 00:00:00 2001 From: Andrea Terzani Date: Wed, 18 Dec 2024 12:16:23 +0000 Subject: [PATCH 7/8] Updated ExternalAgentSolver.java --- .../hermione/stepSolvers/ExternalAgentSolver.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java index d40b9cb..0a83e05 100644 --- a/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java +++ b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java @@ -95,7 +95,7 @@ public class ExternalAgentSolver extends StepSolver { } - int maxTries = 100; + int maxTries = 500; // Pool the status GET api until it return the SUCCESS or FAILED message while(!jsonResponse.get("status").equals("COMPLETED") && !jsonResponse.get("status").equals("FAILED") && maxTries > 0){ response = restTemplate.exchange( @@ -107,16 +107,18 @@ public class ExternalAgentSolver extends StepSolver { jsonResponse = new JSONObject(response.getBody()); try { - Thread.sleep(5000); + Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.error("Thread was interrupted", e); } - + logger.info ("Check status => Remaining tryes :" + maxTries) maxTries--; } - + + logger.info ("Stop pooling agents pod. Latest status = "+jsonResponse.get("status")); + if(jsonResponse.get("status").equals("COMPLETED")){ response = restTemplate.exchange( this.agent_base_url + "/execution_result/" + this.scenarioExecution.getId(), @@ -130,6 +132,8 @@ public class ExternalAgentSolver extends StepSolver { this.scenarioExecution.setNextStepId(this.step.getNextStepId()); }else{ + logger.error ("ERROR on pooling Agents"); + throw new Exception("Agent execution failed with status: " + jsonResponse.get("status")); } From 25b98d768c03a6471b5ac3a2bb913b1ebccbff66 Mon Sep 17 00:00:00 2001 From: Andrea Terzani Date: Wed, 18 Dec 2024 12:18:01 +0000 Subject: [PATCH 8/8] Updated ExternalAgentSolver.java --- .../com/olympus/hermione/stepSolvers/ExternalAgentSolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java index 0a83e05..e88eef2 100644 --- a/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java +++ b/src/main/java/com/olympus/hermione/stepSolvers/ExternalAgentSolver.java @@ -112,7 +112,7 @@ public class ExternalAgentSolver extends StepSolver { Thread.currentThread().interrupt(); logger.error("Thread was interrupted", e); } - logger.info ("Check status => Remaining tryes :" + maxTries) + logger.info ("Check status => Remaining tryes :" + maxTries); maxTries--; }