From 988b2e14b087a81cf614832572fc38bbfa21b41f Mon Sep 17 00:00:00 2001 From: sumedh Date: Wed, 6 Nov 2024 12:06:35 +0530 Subject: [PATCH 01/20] azure-pipelines code added --- apollo-deployment.yaml | 48 +++++++++++++++++++ azure-pipelines.yml | 103 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 apollo-deployment.yaml create mode 100644 azure-pipelines.yml diff --git a/apollo-deployment.yaml b/apollo-deployment.yaml new file mode 100644 index 0000000..90a57d6 --- /dev/null +++ b/apollo-deployment.yaml @@ -0,0 +1,48 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: apollo + namespace: olympus +spec: + replicas: 1 + selector: + matchLabels: + app: apollo + template: + metadata: + labels: + app: apollo + spec: + volumes: + - name: apollo-pv-storage + persistentVolumeClaim: + claimName: private-azurefile-pvc + containers: + - name: apollo + image: olytest/apollo:2ac8bc10 + envFrom: + - configMapRef: + name: olympus-db-shared-cfg + env: + - name: DOCUMENT_UPLOAD_PATH + value: /mnt/apollo_storage/documents + - name: REPOSITORY_UPLOAD_PATH + value: /mnt/apollo_storage/repository + - name: GITLAB_CLOUD_TOKEN + value: glpat-z-72kM9b9yvqd8arvEX1 + - name: GITLAB_ONPREMISES_TOKEN + value: VScokCa7anraKbXisnwe + - name: APOLLO_FE_URL + value: http://apollo-aks.olympusai.live + - name: EUREKA_URI + value: >- + http://eureka-server-service.olympus.svc.cluster.local:8761/eureka + resources: {} + volumeMounts: + - name: apollo-pv-storage + mountPath: /mnt/apollo_storage/documents + subPath: documents + - name: apollo-pv-storage + mountPath: /mnt/apollo_storage/repository + subPath: repository + imagePullPolicy: Always \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..d07d27b --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,103 @@ +# Build your Java project and run tests with Apache Maven. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/java + +trigger: +- develop + +pool: + name: azureselfhosted + +variables: + dockerRegistryServiceConnection: 'docker-registry-olympus' + imageName: 'apollo' # Replace with your Docker image name + shortCommitSha: '' # Initialize variable for short commit SHA + +resources: + repositories: + - repository: olympus-common # Alias for the first repository + type: git + name: Olympus/olympus-common # Project and repository name + ref: master + +steps: +- checkout: self # Check out the pipeline repository (optional) +- checkout: olympus-common # Check out the first repository + +# Extract the short commit SHA +- script: | + echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-7)" + displayName: 'Set Short Commit SHA' + +- script: | + echo "Building the first repository..." + ls -lrth + displayName: 'List olympus-common' + +- task: Maven@3 + inputs: + mavenPomFile: 'olympus-common/pom.xml' + goals: 'install' + # options: '-s $(mvnSettings.secureFilePath)' + #mavenAuthenticateFeed: true + #publishJUnitResults: true + #testResultsFiles: '**/TEST-*.xml' + #javaHomeOption: 'JDKVersion' + #jdkVersionOption: '1.21' + #mavenVersionOption: 'Default' + displayName: 'Build olympus-common' + +- task: Maven@3 + inputs: + mavenPomFile: 'apollo/pom.xml' + goals: 'install' + # options: '-s $(mvnSettings.secureFilePath)' + #mavenAuthenticateFeed: true + #publishJUnitResults: true + #testResultsFiles: '**/TEST-*.xml' + #javaHomeOption: 'JDKVersion' + #jdkVersionOption: '1.21' + #mavenVersionOption: 'Default' + displayName: 'Build apollo' + +- script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images1' + +- task: Docker@2 + inputs: + command: 'buildAndPush' + containerRegistry: $(dockerRegistryServiceConnection) + repository: '$(imageName)' + tags: '$(shortCommitSha)' + Dockerfile: 'apollo/Dockerfile' + +- script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images' + +- script: | + echo "Updating imageName in dpeloyment.yaml" + echo "Before Substitution" + cat apollo/apollo-deployment.yaml + echo "###############################################" + echo "current shell: $SHELL" + echo "imagename bracket : $(imageName)" + echo "###############################################" + sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml + echo "After Substitution" + cat apollo/apollo-deployment.yaml + displayName: 'Update ImageName in YAML file' + +- script: | + echo "deployment using kubectl" + kubectl apply -f apollo/apollo-deployment.yaml + displayName: 'deployment using kubectl' + +- script: | + echo "remove the docker images..." + docker rmi $(docker images -q) + displayName: 'remove docker images' + From 55265a241a356ee06384fe2a39bccb3924d37c29 Mon Sep 17 00:00:00 2001 From: sumedh Date: Wed, 6 Nov 2024 12:15:13 +0530 Subject: [PATCH 02/20] azure-pipelines deleted --- azure-pipelines.yml | 103 -------------------------------------------- 1 file changed, 103 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index d07d27b..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,103 +0,0 @@ -# Build your Java project and run tests with Apache Maven. -# Add steps that analyze code, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/java - -trigger: -- develop - -pool: - name: azureselfhosted - -variables: - dockerRegistryServiceConnection: 'docker-registry-olympus' - imageName: 'apollo' # Replace with your Docker image name - shortCommitSha: '' # Initialize variable for short commit SHA - -resources: - repositories: - - repository: olympus-common # Alias for the first repository - type: git - name: Olympus/olympus-common # Project and repository name - ref: master - -steps: -- checkout: self # Check out the pipeline repository (optional) -- checkout: olympus-common # Check out the first repository - -# Extract the short commit SHA -- script: | - echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-7)" - displayName: 'Set Short Commit SHA' - -- script: | - echo "Building the first repository..." - ls -lrth - displayName: 'List olympus-common' - -- task: Maven@3 - inputs: - mavenPomFile: 'olympus-common/pom.xml' - goals: 'install' - # options: '-s $(mvnSettings.secureFilePath)' - #mavenAuthenticateFeed: true - #publishJUnitResults: true - #testResultsFiles: '**/TEST-*.xml' - #javaHomeOption: 'JDKVersion' - #jdkVersionOption: '1.21' - #mavenVersionOption: 'Default' - displayName: 'Build olympus-common' - -- task: Maven@3 - inputs: - mavenPomFile: 'apollo/pom.xml' - goals: 'install' - # options: '-s $(mvnSettings.secureFilePath)' - #mavenAuthenticateFeed: true - #publishJUnitResults: true - #testResultsFiles: '**/TEST-*.xml' - #javaHomeOption: 'JDKVersion' - #jdkVersionOption: '1.21' - #mavenVersionOption: 'Default' - displayName: 'Build apollo' - -- script: | - echo "Listing the docker images..." - docker images - displayName: 'List docker images1' - -- task: Docker@2 - inputs: - command: 'buildAndPush' - containerRegistry: $(dockerRegistryServiceConnection) - repository: '$(imageName)' - tags: '$(shortCommitSha)' - Dockerfile: 'apollo/Dockerfile' - -- script: | - echo "Listing the docker images..." - docker images - displayName: 'List docker images' - -- script: | - echo "Updating imageName in dpeloyment.yaml" - echo "Before Substitution" - cat apollo/apollo-deployment.yaml - echo "###############################################" - echo "current shell: $SHELL" - echo "imagename bracket : $(imageName)" - echo "###############################################" - sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml - echo "After Substitution" - cat apollo/apollo-deployment.yaml - displayName: 'Update ImageName in YAML file' - -- script: | - echo "deployment using kubectl" - kubectl apply -f apollo/apollo-deployment.yaml - displayName: 'deployment using kubectl' - -- script: | - echo "remove the docker images..." - docker rmi $(docker images -q) - displayName: 'remove docker images' - From 8f15384a8a95a7d47cf7b91b9f5dc12cec0c187b Mon Sep 17 00:00:00 2001 From: sumedh Date: Wed, 6 Nov 2024 12:56:33 +0530 Subject: [PATCH 03/20] apollo-deployment modified --- apollo-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo-deployment.yaml b/apollo-deployment.yaml index 90a57d6..551db1f 100644 --- a/apollo-deployment.yaml +++ b/apollo-deployment.yaml @@ -19,7 +19,7 @@ spec: claimName: private-azurefile-pvc containers: - name: apollo - image: olytest/apollo:2ac8bc10 + image: olympusreg.azurecr.io/apollo:2ac8bc10 envFrom: - configMapRef: name: olympus-db-shared-cfg From 41cca42a65c67e65ba1dc6c1b3068e55bf9bce40 Mon Sep 17 00:00:00 2001 From: sumedh Date: Wed, 6 Nov 2024 12:58:00 +0530 Subject: [PATCH 04/20] tika document reader version removed --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a17aa6..a1ba095 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,6 @@ org.springframework.ai spring-ai-tika-document-reader - 1.0.0-SNAPSHOT From 026f1df1f88050cfadd78ac4d7c8dbf756a36c38 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Wed, 6 Nov 2024 07:29:56 +0000 Subject: [PATCH 05/20] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..c0085a5 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,87 @@ +# Build your Java project and run tests with Apache Maven. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/java +trigger: +- develop +pool: + name: azureselfhosted +variables: + dockerRegistryServiceConnection: 'docker-registry-olympus' + imageName: 'apollo' # Replace with your Docker image name + shortCommitSha: '' # Initialize variable for short commit SHA +resources: + repositories: + - repository: olympus-common # Alias for the first repository + type: git + name: Olympus/olympus-common # Project and repository name + ref: master +steps: +- checkout: self # Check out the pipeline repository (optional) +- checkout: olympus-common # Check out the first repository +# Extract the short commit SHA +- script: | + echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-7)" + displayName: 'Set Short Commit SHA' +- script: | + echo "Building the first repository..." + ls -lrth + displayName: 'List olympus-common' +- task: Maven@3 + inputs: + mavenPomFile: 'olympus-common/pom.xml' + goals: 'install' + # options: '-s $(mvnSettings.secureFilePath)' + #mavenAuthenticateFeed: true + #publishJUnitResults: true + #testResultsFiles: '**/TEST-*.xml' + #javaHomeOption: 'JDKVersion' + #jdkVersionOption: '1.21' + #mavenVersionOption: 'Default' + displayName: 'Build olympus-common' +- task: Maven@3 + inputs: + mavenPomFile: 'apollo/pom.xml' + goals: 'install' + # options: '-s $(mvnSettings.secureFilePath)' + #mavenAuthenticateFeed: true + #publishJUnitResults: true + #testResultsFiles: '**/TEST-*.xml' + #javaHomeOption: 'JDKVersion' + #jdkVersionOption: '1.21' + #mavenVersionOption: 'Default' + displayName: 'Build apollo' +- script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images1' +- task: Docker@2 + inputs: + command: 'buildAndPush' + containerRegistry: $(dockerRegistryServiceConnection) + repository: '$(imageName)' + tags: '$(shortCommitSha)' + Dockerfile: 'apollo/Dockerfile' +- script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images' +- script: | + echo "Updating imageName in dpeloyment.yaml" + echo "Before Substitution" + cat apollo/apollo-deployment.yaml + echo "###############################################" + echo "current shell: $SHELL" + echo "imagename bracket : $(imageName)" + echo "###############################################" + sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml + echo "After Substitution" + cat apollo/apollo-deployment.yaml + displayName: 'Update ImageName in YAML file' +- script: | + echo "deployment using kubectl" + kubectl apply -f apollo/apollo-deployment.yaml + displayName: 'deployment using kubectl' +- script: | + echo "remove the docker images..." + docker rmi $(docker images -q) + displayName: 'remove docker images' \ No newline at end of file From 147b44d93110fb12391de26aaa20aa43ad292ccd Mon Sep 17 00:00:00 2001 From: sumedh Date: Wed, 6 Nov 2024 13:03:25 +0530 Subject: [PATCH 06/20] azure pipeline modified --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c0085a5..ce0c0de 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,7 +2,7 @@ # Add steps that analyze code, save build artifacts, deploy, and more: # https://docs.microsoft.com/azure/devops/pipelines/languages/java trigger: -- develop +- master pool: name: azureselfhosted variables: @@ -41,7 +41,7 @@ steps: - task: Maven@3 inputs: mavenPomFile: 'apollo/pom.xml' - goals: 'install' + goals: 'install -DskipTests' # options: '-s $(mvnSettings.secureFilePath)' #mavenAuthenticateFeed: true #publishJUnitResults: true From ba515f9de9a3ae9d3608304c0629b2be2f042d0e Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:15:21 +0000 Subject: [PATCH 07/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 176 +++++++++++++++++++++++--------------------- 1 file changed, 92 insertions(+), 84 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ce0c0de..92b625e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,87 +1,95 @@ -# Build your Java project and run tests with Apache Maven. -# Add steps that analyze code, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/java trigger: -- master -pool: - name: azureselfhosted + - master + variables: dockerRegistryServiceConnection: 'docker-registry-olympus' - imageName: 'apollo' # Replace with your Docker image name - shortCommitSha: '' # Initialize variable for short commit SHA -resources: - repositories: - - repository: olympus-common # Alias for the first repository - type: git - name: Olympus/olympus-common # Project and repository name - ref: master -steps: -- checkout: self # Check out the pipeline repository (optional) -- checkout: olympus-common # Check out the first repository -# Extract the short commit SHA -- script: | - echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-7)" - displayName: 'Set Short Commit SHA' -- script: | - echo "Building the first repository..." - ls -lrth - displayName: 'List olympus-common' -- task: Maven@3 - inputs: - mavenPomFile: 'olympus-common/pom.xml' - goals: 'install' - # options: '-s $(mvnSettings.secureFilePath)' - #mavenAuthenticateFeed: true - #publishJUnitResults: true - #testResultsFiles: '**/TEST-*.xml' - #javaHomeOption: 'JDKVersion' - #jdkVersionOption: '1.21' - #mavenVersionOption: 'Default' - displayName: 'Build olympus-common' -- task: Maven@3 - inputs: - mavenPomFile: 'apollo/pom.xml' - goals: 'install -DskipTests' - # options: '-s $(mvnSettings.secureFilePath)' - #mavenAuthenticateFeed: true - #publishJUnitResults: true - #testResultsFiles: '**/TEST-*.xml' - #javaHomeOption: 'JDKVersion' - #jdkVersionOption: '1.21' - #mavenVersionOption: 'Default' - displayName: 'Build apollo' -- script: | - echo "Listing the docker images..." - docker images - displayName: 'List docker images1' -- task: Docker@2 - inputs: - command: 'buildAndPush' - containerRegistry: $(dockerRegistryServiceConnection) - repository: '$(imageName)' - tags: '$(shortCommitSha)' - Dockerfile: 'apollo/Dockerfile' -- script: | - echo "Listing the docker images..." - docker images - displayName: 'List docker images' -- script: | - echo "Updating imageName in dpeloyment.yaml" - echo "Before Substitution" - cat apollo/apollo-deployment.yaml - echo "###############################################" - echo "current shell: $SHELL" - echo "imagename bracket : $(imageName)" - echo "###############################################" - sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml - echo "After Substitution" - cat apollo/apollo-deployment.yaml - displayName: 'Update ImageName in YAML file' -- script: | - echo "deployment using kubectl" - kubectl apply -f apollo/apollo-deployment.yaml - displayName: 'deployment using kubectl' -- script: | - echo "remove the docker images..." - docker rmi $(docker images -q) - displayName: 'remove docker images' \ No newline at end of file + imageName: 'apollo' + shortCommitSha: '' + envName: 'aks-olympus' + +stages: + - stage: Build + displayName: Build stage + jobs: + - job: Build + displayName: Build + pool: + name: azureselfhosted + steps: + - checkout: self + + - script: | + echo "##vso[task.setvariable variable=shortCommitSha]$(Build.SourceVersion.SHA.Substring(0, 8))" + displayName: 'Set Short Commit SHA' + + - script: | + echo "Building the first repository..." + ls -lrth + displayName: 'List olympus-common' + + - task: Maven@3 + inputs: + mavenPomFile: 'olympus-common/pom.xml' + goals: 'install' + displayName: 'Build olympus-common' + + - task: Maven@3 + inputs: + mavenPomFile: 'apollo/pom.xml' + goals: 'install -DskipTests' + displayName: 'Build apollo' + + - script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images1' + + - task: Docker@2 + inputs: + command: 'buildAndPush' + containerRegistry: $(dockerRegistryServiceConnection) + repository: '$(imageName)' + tags: '$(shortCommitSha)' + Dockerfile: 'apollo/Dockerfile' + + - script: | + echo "Listing the docker images..." + docker images + displayName: 'List docker images' + + - stage: Deploy + displayName: Deploy stage + jobs: + - deployment: Deploy + displayName: Deploy to $(envName) + condition: succeeded() + pool: + name: azureselfhosted + environment: $(envName) + strategy: + runOnce: + deploy: + steps: + - script: | + echo "Updating imageName in dpeloyment.yaml" + echo "Before Substitution" + cat apollo/apollo-deployment.yaml + echo "###############################################" + echo "current shell: $SHELL" + echo "imagename bracket : $(imageName)" + echo "###############################################" + sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml + echo "After Substitution" + cat apollo/apollo-deployment.yaml + displayName: 'Update ImageName in YAML file' + + - script: | + echo "deployment using kubectl" + kubectl apply -f apollo/apollo-deployment.yaml + displayName: 'Deployment using kubectl' + + - script: | + echo "Removing Docker images to clean up..." + docker rmi $(docker images -q) + displayName: 'Clean up Docker Images' + continueOnError: true From cbdfd800b0883bd0f0419b4ba9d5136e4ab815a3 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:16:41 +0000 Subject: [PATCH 08/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 92b625e..27f180a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ stages: - checkout: self - script: | - echo "##vso[task.setvariable variable=shortCommitSha]$(Build.SourceVersion.SHA.Substring(0, 8))" + echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-8)" displayName: 'Set Short Commit SHA' - script: | From 28260a796ecf72ac4d8b0a9be3da12ba7091af15 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:18:03 +0000 Subject: [PATCH 09/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 27f180a..23775c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,6 +1,13 @@ trigger: - master +resources: + repositories: + - repository: olympus-common # Alias for the first repository + type: git + name: Olympus/olympus-common # Project and repository name + ref: master + variables: dockerRegistryServiceConnection: 'docker-registry-olympus' imageName: 'apollo' @@ -17,6 +24,7 @@ stages: name: azureselfhosted steps: - checkout: self + - checkout: olympus-common - script: | echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-8)" From 3928033662e37818c059d615ef107d85980c0dcd Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:30:47 +0000 Subject: [PATCH 10/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 23775c5..46d8b9f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -78,26 +78,25 @@ stages: runOnce: deploy: steps: + - checkout: self + - script: | - echo "Updating imageName in dpeloyment.yaml" + echo $(shortCommitSha) + echo "Updating imageName in deployment.yaml" echo "Before Substitution" - cat apollo/apollo-deployment.yaml + cat apollo-deployment.yaml echo "###############################################" echo "current shell: $SHELL" echo "imagename bracket : $(imageName)" echo "###############################################" - sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo/apollo-deployment.yaml + sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo-deployment.yaml echo "After Substitution" - cat apollo/apollo-deployment.yaml + cat apollo-deployment.yaml displayName: 'Update ImageName in YAML file' - - script: | - echo "deployment using kubectl" - kubectl apply -f apollo/apollo-deployment.yaml - displayName: 'Deployment using kubectl' - - script: | echo "Removing Docker images to clean up..." docker rmi $(docker images -q) displayName: 'Clean up Docker Images' continueOnError: true + condition: always() From 4f4b0751b0f24d39138c65ffd303a26ce32cfc3a Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:34:32 +0000 Subject: [PATCH 11/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 46d8b9f..5cd6564 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -80,6 +80,10 @@ stages: steps: - checkout: self + - script: | + echo "##vso[task.setvariable variable=shortCommitSha]$(echo $(Build.SourceVersion) | cut -c1-8)" + displayName: 'Set Short Commit SHA' + - script: | echo $(shortCommitSha) echo "Updating imageName in deployment.yaml" @@ -94,6 +98,11 @@ stages: cat apollo-deployment.yaml displayName: 'Update ImageName in YAML file' + - script: | + echo "deployment using kubectl" + kubectl apply -f apollo-deployment.yaml + displayName: 'Deployment using kubectl' + - script: | echo "Removing Docker images to clean up..." docker rmi $(docker images -q) From ef44b61aeb75647ab262be48c7491e68efdfe918 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:49:23 +0000 Subject: [PATCH 12/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5cd6564..e718281 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -69,7 +69,7 @@ stages: displayName: Deploy stage jobs: - deployment: Deploy - displayName: Deploy to $(envName) + displayName: "Deploy to $(envName)" condition: succeeded() pool: name: azureselfhosted From 4b3c76a8df65491d3ebf724fd7fe7dad825d304b Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 07:58:21 +0000 Subject: [PATCH 13/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e718281..48510c9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,7 +12,7 @@ variables: dockerRegistryServiceConnection: 'docker-registry-olympus' imageName: 'apollo' shortCommitSha: '' - envName: 'aks-olympus' + envName: "aks-olympus" stages: - stage: Build @@ -69,7 +69,7 @@ stages: displayName: Deploy stage jobs: - deployment: Deploy - displayName: "Deploy to $(envName)" + displayName: "Deploy to ${{variables.envName}}" condition: succeeded() pool: name: azureselfhosted @@ -108,4 +108,4 @@ stages: docker rmi $(docker images -q) displayName: 'Clean up Docker Images' continueOnError: true - condition: always() + condition: always() \ No newline at end of file From f6899747277f8d070ecfc13d32983058523484e8 Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 08:01:26 +0000 Subject: [PATCH 14/20] working version of pipeline --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 48510c9..ad5b550 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ stages: displayName: Build stage jobs: - job: Build - displayName: Build + displayName: "Build & push to ACR" pool: name: azureselfhosted steps: @@ -108,4 +108,4 @@ stages: docker rmi $(docker images -q) displayName: 'Clean up Docker Images' continueOnError: true - condition: always() \ No newline at end of file + condition: always() From c80ef79357c32f5c2e95ba445bb9c5e8208e4be4 Mon Sep 17 00:00:00 2001 From: sumedh Date: Fri, 8 Nov 2024 13:55:10 +0530 Subject: [PATCH 15/20] manifests file uploaded --- apollo-deployment.yaml => manifests/apollo-deployment.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apollo-deployment.yaml => manifests/apollo-deployment.yaml (100%) diff --git a/apollo-deployment.yaml b/manifests/apollo-deployment.yaml similarity index 100% rename from apollo-deployment.yaml rename to manifests/apollo-deployment.yaml From dc1d679a0e44d28cef84cb0af9d8bb1dd6f9ef0a Mon Sep 17 00:00:00 2001 From: "shamrao shinde, sumedh" Date: Fri, 8 Nov 2024 08:43:13 +0000 Subject: [PATCH 16/20] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ad5b550..3ab3396 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,7 +12,9 @@ variables: dockerRegistryServiceConnection: 'docker-registry-olympus' imageName: 'apollo' shortCommitSha: '' - envName: "aks-olympus" + envName: 'aks-olympus' + azurecontainerRegistry: 'olympusreg.azurecr.io' + resourceName: 'olympus' stages: - stage: Build @@ -33,7 +35,7 @@ stages: - script: | echo "Building the first repository..." ls -lrth - displayName: 'List olympus-common' + displayName: 'list files' - task: Maven@3 inputs: @@ -50,30 +52,38 @@ stages: - script: | echo "Listing the docker images..." docker images - displayName: 'List docker images1' + displayName: 'list docker images' - task: Docker@2 inputs: command: 'buildAndPush' - containerRegistry: $(dockerRegistryServiceConnection) repository: '$(imageName)' - tags: '$(shortCommitSha)' Dockerfile: 'apollo/Dockerfile' + containerRegistry: $(dockerRegistryServiceConnection) + tags: '$(shortCommitSha)' + displayName: 'Build and push an image to container registry' - script: | echo "Listing the docker images..." docker images displayName: 'List docker images' + + - upload: apollo/manifests + artifact: manifests - stage: Deploy displayName: Deploy stage + dependsOn: Build + jobs: - deployment: Deploy - displayName: "Deploy to ${{variables.envName}}" condition: succeeded() + displayName: "Deploy to ${{variables.envName}}" pool: name: azureselfhosted - environment: $(envName) + environment: + name: $(envName).$(resourceName) + resourceType: Kubernetes strategy: runOnce: deploy: @@ -87,21 +97,16 @@ stages: - script: | echo $(shortCommitSha) echo "Updating imageName in deployment.yaml" - echo "Before Substitution" - cat apollo-deployment.yaml - echo "###############################################" - echo "current shell: $SHELL" - echo "imagename bracket : $(imageName)" - echo "###############################################" - sed -i "s|image: olympusreg.azurecr.io/$(imageName):.*|image: olympusreg.azurecr.io/$(imageName):$(shortCommitSha)|" apollo-deployment.yaml - echo "After Substitution" - cat apollo-deployment.yaml displayName: 'Update ImageName in YAML file' - - script: | - echo "deployment using kubectl" - kubectl apply -f apollo-deployment.yaml - displayName: 'Deployment using kubectl' + - task: KubernetesManifest@0 + displayName: Deploy to Kubernetes cluster + inputs: + action: deploy + manifests: | + $(Pipeline.Workspace)/manifests/apollo-deployment.yaml + containers: | + $(azurecontainerRegistry)/$(imageName):$(shortCommitSha) - script: | echo "Removing Docker images to clean up..." From 7a3881e6dcf51ff312a594c720680b33729c7478 Mon Sep 17 00:00:00 2001 From: "DIR\\maria.del.valle" Date: Mon, 11 Nov 2024 17:35:10 +0100 Subject: [PATCH 17/20] added limit embed documents --- .../olympus/apollo/services/KSIngestor.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 28e92ce..5463a7b 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -15,6 +15,7 @@ import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; @@ -37,7 +38,12 @@ public class KSIngestor { @Autowired private VectorStore vectorStore; + + @Value("${ksingestor.embedded.doc.batch.size:20}") + private int embDocsBatchSize; + @Value("${ksingestor.embedded.doc.retry.time:20000}") + private int embDocRetryTime; Logger logger = LoggerFactory.getLogger(KSIngestor.class); @@ -209,12 +215,20 @@ public class KSIngestor { logger.info("Embedding documents"); docs.forEach(doc -> logger.info("Document metadata: " + doc.getMetadata())); - try { - vectorStore.add(docs); - logger.info("Documents embedded"); - } catch (Exception e) { - logger.error("Error embedding documents: ", e); + + int batchSize = embDocsBatchSize; + for (int i = 0; i < docs.size(); i += batchSize) { + int end = Math.min(i + batchSize, docs.size()); + List currentList = docs.subList(i, end); + try { + Thread.sleep(embDocRetryTime); + vectorStore.add(currentList); + logger.info("Documents embedded - Progress: Batch from {} to {} completed", i, end); + } catch (Exception e) { + logger.error("Error embedding documents from {} to {}: {}", i, end, e.getMessage()); + } } + } public List testSimilaritySearch(String query,String filterQuery) { From 88c59bfbef4d9ab7df1b590181671567952ef456 Mon Sep 17 00:00:00 2001 From: sumedh Date: Tue, 12 Nov 2024 11:52:26 +0530 Subject: [PATCH 18/20] deployment file modified --- manifests/apollo-deployment.yaml | 18 ++++-------------- src/main/resources/application.yml | 29 +++++++---------------------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/manifests/apollo-deployment.yaml b/manifests/apollo-deployment.yaml index 551db1f..3529b6e 100644 --- a/manifests/apollo-deployment.yaml +++ b/manifests/apollo-deployment.yaml @@ -23,20 +23,10 @@ spec: envFrom: - configMapRef: name: olympus-db-shared-cfg - env: - - name: DOCUMENT_UPLOAD_PATH - value: /mnt/apollo_storage/documents - - name: REPOSITORY_UPLOAD_PATH - value: /mnt/apollo_storage/repository - - name: GITLAB_CLOUD_TOKEN - value: glpat-z-72kM9b9yvqd8arvEX1 - - name: GITLAB_ONPREMISES_TOKEN - value: VScokCa7anraKbXisnwe - - name: APOLLO_FE_URL - value: http://apollo-aks.olympusai.live - - name: EUREKA_URI - value: >- - http://eureka-server-service.olympus.svc.cluster.local:8761/eureka + - configMapRef: + name: olympus-ai-shared-cfg + - configMapRef: + name: olympus-common-shared-cfg resources: {} volumeMounts: - name: apollo-pv-storage diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 9bbf31d..6a638e2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -5,7 +5,7 @@ apollo: fe: url: "http://localhost:5173" java-parser-module: - url: "http://java-parser-module-service.olympus.svc.cluster.local:8080" + url: "http://localhost:8084" spring: application: name: apollo @@ -13,37 +13,30 @@ spring: azure: openai: endpoint: "https://ai-olympus.openai.azure.com/" - api-key: "9fb33cc69d914d4c8225b974876510b5" + api-key: "9fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" vectorstore: azure: - api-key: "jxKqZvbMKuo1MwXs8ilEAeRDeswtoTXO1lWX600jP2AzSeDXo1nq" + api-key: "jxKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" url: "https://search-olympus.search.windows.net" initialize-schema: true data: mongodb: - uri: mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus - database: olympus - username: olympus_adm - password: 26111979 + uri: + database: + username: + password: servlet: multipart: max-file-size: 5000000MB max-request-size: 500000MB -#path to the repository -ingestion: - repository: - basepath: C:\\Users\\andrea.terzani\\dev\\Olympus gitlab: - token: "xxxxxxxx" path: /mnt/apollo_storage/repository #C:\\repos\\olympus_ai\\gitClone cloud: - url: "https://gi2tlab.com/api/v4" token: "xxxxxxxx" onpremises: - url: "http://localhost:8081/api" token: "xxxxxxxx" eureka: @@ -53,14 +46,6 @@ eureka: instance: preferIpAddress: true -#spring.jpa.show-sql=true -#spring.jpa.hibernate.ddl-auto=update -#spring.datasource.url=jdbc:postgresql://localhost:5432/olympus -#spring.datasource.username=andreaterzani -#spring.datasource.password=26111979 -#spring.datasource.driver-class-name=org.postgresql.Driver -#spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect - logging: level: root: INFO From 2bb0108bbca75af589e29cbeeca13b4a2993d2e0 Mon Sep 17 00:00:00 2001 From: "andrea.terzani" Date: Tue, 26 Nov 2024 11:59:29 +0100 Subject: [PATCH 19/20] Aggiunta della classe VectorSearchRequest e implementazione del controller SearchDocController per la ricerca vettoriale. Aggiornamento della configurazione di sicurezza e delle dipendenze nel pom.xml. Modifiche al file application.yml per la configurazione del servizio Azure. --- pom.xml | 9 +++- .../olympus/apollo/config/ApolloConfig.java | 44 +++++++++++++++ ...dingConfig.java => VectorStoreConfig.java} | 41 +++++++------- .../controllers/SearchDocController.java | 53 +++++++++++++++++++ .../apollo/dto/VectorSearchRequest.java | 20 +++++++ .../security/config/SecurityConfig.java | 12 ++--- .../olympus/apollo/services/KSIngestor.java | 1 + src/main/resources/application.yml | 26 ++++++--- 8 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 src/main/java/com/olympus/apollo/config/ApolloConfig.java rename src/main/java/com/olympus/apollo/config/{EmbeddingConfig.java => VectorStoreConfig.java} (72%) create mode 100644 src/main/java/com/olympus/apollo/controllers/SearchDocController.java create mode 100644 src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java diff --git a/pom.xml b/pom.xml index a1ba095..5e060a7 100644 --- a/pom.xml +++ b/pom.xml @@ -62,12 +62,12 @@ org.springframework.ai spring-ai-mongodb-atlas-store-spring-boot-starter - +--> org.springframework.ai spring-ai-openai-spring-boot-starter - --> + org.springframework.ai @@ -79,6 +79,11 @@ spring-ai-azure-store + + org.springframework.ai + spring-ai-chroma-store + + org.springframework.cloud spring-cloud-starter-netflix-eureka-client diff --git a/src/main/java/com/olympus/apollo/config/ApolloConfig.java b/src/main/java/com/olympus/apollo/config/ApolloConfig.java new file mode 100644 index 0000000..516bf18 --- /dev/null +++ b/src/main/java/com/olympus/apollo/config/ApolloConfig.java @@ -0,0 +1,44 @@ +package com.olympus.apollo.config; + +import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.openai.OpenAiEmbeddingModel; +import org.springframework.ai.vectorstore.ChromaVectorStore; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.azure.AzureVectorStore; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +public class ApolloConfig { + + private final OpenAiEmbeddingModel openAiEmbeddingModel; + private final AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel; + + + // Autowiring beans from both libraries + public ApolloConfig(OpenAiEmbeddingModel myServiceLib1, + AzureOpenAiEmbeddingModel azureOpenAiEmbeddingModel) { + this.openAiEmbeddingModel = myServiceLib1; + this.azureOpenAiEmbeddingModel = azureOpenAiEmbeddingModel; + + + } + + // Re-declaring and marking one as primary + @Bean + public EmbeddingModel primaryMyService() { + return openAiEmbeddingModel; // Choose the one you want as primary + } + + // Optionally declare the other bean without primary + @Bean + @Primary + public EmbeddingModel secondaryMyService() { + return azureOpenAiEmbeddingModel; // The other one + } + + + +} diff --git a/src/main/java/com/olympus/apollo/config/EmbeddingConfig.java b/src/main/java/com/olympus/apollo/config/VectorStoreConfig.java similarity index 72% rename from src/main/java/com/olympus/apollo/config/EmbeddingConfig.java rename to src/main/java/com/olympus/apollo/config/VectorStoreConfig.java index d8a2643..1a64a32 100644 --- a/src/main/java/com/olympus/apollo/config/EmbeddingConfig.java +++ b/src/main/java/com/olympus/apollo/config/VectorStoreConfig.java @@ -1,30 +1,22 @@ package com.olympus.apollo.config; -import com.azure.core.credential.AzureKeyCredential; -import com.azure.search.documents.indexes.SearchIndexClient; -import com.azure.search.documents.indexes.SearchIndexClientBuilder; -import org.springframework.ai.azure.openai.AzureOpenAiEmbeddingModel; -import org.springframework.ai.embedding.EmbeddingModel; -/*import org.springframework.ai.openai.OpenAiEmbeddingModel; -import org.springframework.ai.openai.api.OpenAiApi; -import org.springframework.ai.vectorstore.MongoDBAtlasVectorStore;*/ - -import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.ai.vectorstore.azure.AzureVectorStore; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; - import java.util.ArrayList; import java.util.List; -@Configuration -public class EmbeddingConfig { +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.azure.AzureVectorStore; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.azure.core.credential.AzureKeyCredential; +import com.azure.search.documents.indexes.SearchIndexClient; +import com.azure.search.documents.indexes.SearchIndexClientBuilder; + +//@Configuration +public class VectorStoreConfig { @Value("${spring.ai.vectorstore.azure.api-key}") @@ -42,7 +34,7 @@ public class EmbeddingConfig { } @Bean - public VectorStore vectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) { + public VectorStore azureVectorStore(SearchIndexClient searchIndexClient, @Qualifier("azureOpenAiEmbeddingModel") EmbeddingModel embeddingModel) { List fields = new ArrayList<>(); fields.add(AzureVectorStore.MetadataField.text("KsApplicationName")); @@ -56,3 +48,6 @@ public class EmbeddingConfig { } } + + + diff --git a/src/main/java/com/olympus/apollo/controllers/SearchDocController.java b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java new file mode 100644 index 0000000..49392bf --- /dev/null +++ b/src/main/java/com/olympus/apollo/controllers/SearchDocController.java @@ -0,0 +1,53 @@ +package com.olympus.apollo.controllers; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.LoggerFactory; +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.olympus.apollo.dto.VectorSearchRequest; + +import ch.qos.logback.classic.Logger; + +@RestController +@RequestMapping("/api/vsearch") + +public class SearchDocController { + + @Autowired + private VectorStore vectorStore; + + Logger logger = (Logger) LoggerFactory.getLogger(SearchDocController.class); + + @PostMapping("/doc_search") + public List vectorSearch(@RequestBody VectorSearchRequest vectorSearchRequest) { + + SearchRequest request = SearchRequest.defaults() + .withQuery(vectorSearchRequest.getQuery()) + .withTopK(vectorSearchRequest.getTopK()) + .withSimilarityThreshold(vectorSearchRequest.getThreshold()); + + if(vectorSearchRequest.getFilterQuery() != null && !vectorSearchRequest.getFilterQuery().isEmpty()){ + request.withFilterExpression(vectorSearchRequest.getFilterQuery()); + logger.info("Using Filter expression: " + vectorSearchRequest.getFilterQuery()); + } + + List docs = this.vectorStore.similaritySearch(request); + logger.info("Number of VDB retrieved documents: " + docs.size()); + + List results = new ArrayList<>(); + for(Document doc : docs){ + results.add(doc.getContent()); + } + return results; + + } +} diff --git a/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java b/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java new file mode 100644 index 0000000..c3bea71 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/VectorSearchRequest.java @@ -0,0 +1,20 @@ +package com.olympus.apollo.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class VectorSearchRequest { + + + private String query; + private String filterQuery; + private int topK; + private float threshold; + +} diff --git a/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java b/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java index 195b422..50e215f 100644 --- a/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java +++ b/src/main/java/com/olympus/apollo/security/config/SecurityConfig.java @@ -1,21 +1,14 @@ package com.olympus.apollo.security.config; -import jakarta.servlet.http.HttpServletResponse; -import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import org.springframework.context.annotation.Lazy; import org.springframework.security.authentication.AuthenticationManager; - -import org.springframework.security.authentication.AuthenticationProvider; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.crypto.password.NoOpPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.SecurityFilterChain; @@ -77,8 +70,9 @@ public class SecurityConfig { http.csrf(csrf -> csrf.disable()) .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .authorizeHttpRequests(auth -> auth.requestMatchers("/api/auth/**").permitAll() - .requestMatchers("/api/test/**").permitAll() - .anyRequest().authenticated()); //permitAll()); + .requestMatchers("/api/test/**").permitAll() + .requestMatchers("/**").permitAll() + .anyRequest().authenticated()); //permitAll()); http.authenticationProvider(authenticationProvider()); diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 5463a7b..b277a71 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -232,6 +232,7 @@ public class KSIngestor { } public List testSimilaritySearch(String query,String filterQuery) { + SearchRequest searchRequest = SearchRequest.defaults().withQuery(query).withTopK(5).withSimilarityThreshold(0.1); if(filterQuery!=null && !filterQuery.isEmpty()){ diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6a638e2..be51639 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -12,8 +12,8 @@ spring: ai: azure: openai: - endpoint: "https://ai-olympus.openai.azure.com/" - api-key: "9fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + endpoint: "https://ai-olympus-new.openai.azure.com/" + api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" vectorstore: @@ -21,16 +21,25 @@ spring: api-key: "jxKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" url: "https://search-olympus.search.windows.net" initialize-schema: true + chroma: + client: + host: "http://108.142.74.161" + port: "8000" + key-token: "nVYLh3eq92aJP4x08dNdWngilPG2ooj9" + initialize-schema: "true" + collection-name: "olympus" data: mongodb: - uri: - database: - username: - password: + uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000 + database: olympus + username: olympusadmin + password: Camilla123! servlet: multipart: max-file-size: 5000000MB max-request-size: 500000MB + main: + allow-circular-references: true gitlab: path: /mnt/apollo_storage/repository #C:\\repos\\olympus_ai\\gitClone @@ -50,4 +59,7 @@ logging: level: root: INFO #feign: DEBUG - #org.springframework.web.client: DEBUG \ No newline at end of file + #org.springframework.web.client: DEBUG + +java-re-module: + url: "http://localhost:8084" \ No newline at end of file From 840ae0ba51d8a5fc9a01c54c013c94812ba7b027 Mon Sep 17 00:00:00 2001 From: Florinda Date: Wed, 27 Nov 2024 09:27:52 +0100 Subject: [PATCH 20/20] Project + KsDocument --- .../FeApi/KsDocumentController.java | 46 ++++++++++- .../apollo/controllers/ProjectController.java | 45 +++++++++++ .../repository/KSDocumentRepository.java | 11 +++ .../apollo/repository/ProjectRepository.java | 12 +++ .../olympus/apollo/security/entity/User.java | 12 +++ .../apollo/services/KSDocumentService.java | 78 ++++++++++++++++++ .../apollo/services/ProjectService.java | 80 +++++++++++++++++++ src/main/resources/application.yml | 14 ++-- 8 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/olympus/apollo/controllers/ProjectController.java create mode 100644 src/main/java/com/olympus/apollo/repository/ProjectRepository.java create mode 100644 src/main/java/com/olympus/apollo/services/KSDocumentService.java create mode 100644 src/main/java/com/olympus/apollo/services/ProjectService.java diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java index b26ed49..ac01a41 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java @@ -3,10 +3,21 @@ package com.olympus.apollo.controllers.FeApi; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.UrlResource; import org.springframework.web.bind.annotation.*; import com.olympus.model.apollo.KSDocument; import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.services.KSDocumentService; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import java.nio.file.Files; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +import java.nio.file.Path; +import java.nio.file.Paths; + @RestController @RequestMapping("/fe-api/ksdocuments") @@ -14,12 +25,14 @@ public class KsDocumentController { @Autowired private KSDocumentRepository ksDocumentREpository; - + + @Autowired + private KSDocumentService ksDocumentService; @GetMapping("") public List getDocuments() { - List result = (List) ksDocumentREpository.findAll(); + List result = ksDocumentService.findByProjectNameAndApplicationName(); return result; } @@ -30,4 +43,33 @@ public class KsDocumentController { return result; } + + + /* @PostMapping("/downloadKSDocument") + public ResponseEntity downloadFile(@RequestBody KSDocument doc) { + try { + // Percorso al file + Path filePath = Paths.get(doc.getFilePath()).normalize(); + Resource resource = new UrlResource(filePath.toUri()); + + if (!resource.exists()) { + return ResponseEntity.notFound().build(); + } + + // Configurazione della risposta HTTP + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"") + .body(resource); + } catch (Exception e) { + return ResponseEntity.internalServerError().build(); + } + }*/ + + @PostMapping("/downloadKSDocument") +public ResponseEntity downloadFile(@RequestBody KSDocument doc) { + + return ksDocumentService.downloadKSDocument(doc); + +} + } diff --git a/src/main/java/com/olympus/apollo/controllers/ProjectController.java b/src/main/java/com/olympus/apollo/controllers/ProjectController.java new file mode 100644 index 0000000..5bf9474 --- /dev/null +++ b/src/main/java/com/olympus/apollo/controllers/ProjectController.java @@ -0,0 +1,45 @@ +package com.olympus.apollo.controllers; + +import java.util.Optional; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.apollo.services.ProjectService; +import com.olympus.model.Application; +import com.olympus.model.Project; + +@RestController +public class ProjectController { + + @Autowired + ProjectService projectService; + + @Autowired + ProjectRepository projectRepo; + + @GetMapping("/userProjects") + public Iterable getUserProjects() { + return projectService.getListProjectsByUser(); + } + + @PostMapping("/updateSelectedProject") + public User setSelectedProjects(@RequestBody Project projectId) { + return projectService.updateUserSelectedProject(projectId); + } + + @PostMapping("/updateSelectedApplication") + public User setSelectedApplication(@RequestBody Application application) { + return projectService.updateUserSelectedApplication(application); + } + @PostMapping("/getProject") + public Optional getUserProjects(@RequestBody String projectId) { + return projectRepo.findById(projectId); + } + + +} diff --git a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java index 3e07b81..1f86602 100644 --- a/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java +++ b/src/main/java/com/olympus/apollo/repository/KSDocumentRepository.java @@ -1,7 +1,11 @@ package com.olympus.apollo.repository; +import java.util.List; + +import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.data.rest.core.annotation.RepositoryRestResource; import org.springframework.web.bind.annotation.CrossOrigin; @@ -13,4 +17,11 @@ import com.olympus.model.apollo.KSDocument; public interface KSDocumentRepository extends MongoRepository { public Iterable findAllByIngestionStatus(String status); + + @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0, 'ingestionInfo.metadata.KsApplicationName': ?1 }") + public List findByProjectNameAndApplicationName(String projectName, String applicationName, Sort sort); + + @Query("{ 'ingestionInfo.metadata.KsProjectName': ?0 }") + public List findByProjectName(String projectName, Sort sort); + } diff --git a/src/main/java/com/olympus/apollo/repository/ProjectRepository.java b/src/main/java/com/olympus/apollo/repository/ProjectRepository.java new file mode 100644 index 0000000..34f5371 --- /dev/null +++ b/src/main/java/com/olympus/apollo/repository/ProjectRepository.java @@ -0,0 +1,12 @@ +package com.olympus.apollo.repository; + +import org.bson.types.ObjectId; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; +import com.olympus.model.Project; + +@Repository +public interface ProjectRepository extends MongoRepository { + + +} diff --git a/src/main/java/com/olympus/apollo/security/entity/User.java b/src/main/java/com/olympus/apollo/security/entity/User.java index 0524cac..98763a5 100644 --- a/src/main/java/com/olympus/apollo/security/entity/User.java +++ b/src/main/java/com/olympus/apollo/security/entity/User.java @@ -4,10 +4,14 @@ import java.util.Collection; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.DocumentReference; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.userdetails.UserDetails; +import com.olympus.model.Application; +import com.olympus.model.Project; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -31,6 +35,14 @@ public class User implements UserDetails{ private String name; private String surname; + @DocumentReference + private Project selectedProject; + @DocumentReference + private List lstProjects; + + @DocumentReference + private Application selectedApplication; + private Role role; @Override diff --git a/src/main/java/com/olympus/apollo/services/KSDocumentService.java b/src/main/java/com/olympus/apollo/services/KSDocumentService.java new file mode 100644 index 0000000..d2481d5 --- /dev/null +++ b/src/main/java/com/olympus/apollo/services/KSDocumentService.java @@ -0,0 +1,78 @@ +package com.olympus.apollo.services; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.tomcat.util.openssl.openssl_h; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.model.apollo.KSDocument; + +@Service +public class KSDocumentService { + + private Logger logger = LoggerFactory.getLogger(KSDocumentService.class); + + @Autowired + private KSDocumentRepository ksdocRepo; + + public List findByProjectNameAndApplicationName() { + logger.info("findByProjectNameAndApplicationName function:"); + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + try { + if( principal.getSelectedApplication()==null){ + return ksdocRepo.findByProjectName(principal.getSelectedProject().getInternal_name(), Sort.by(Sort.Direction.DESC, "ingestionDate")); + }else{ + return ksdocRepo.findByProjectNameAndApplicationName(principal.getSelectedProject().getInternal_name(), principal.getSelectedApplication().getInternal_name(), Sort.by(Sort.Direction.DESC, "ingestionDate")); + } + } catch (Exception e) { + logger.error("Error in findByProjectNameAndApplicationName function: " + e.getMessage()); + } + + return null; + } + + public ResponseEntity downloadKSDocument(KSDocument doc) { + + logger.info("downloadKSDocument function:"); + + try { + // Percorso al file + Path filePath = Paths.get(doc.getFilePath()).normalize(); + Resource resource = new UrlResource(filePath.toUri()); + + if (!resource.exists()) { + return ResponseEntity.notFound().build(); + } + + // Determina il tipo MIME dinamicamente + String contentType = Files.probeContentType(filePath); + if (contentType == null) { + // Tipo MIME predefinito se non determinabile + contentType = "application/octet-stream"; + } + + // Configurazione della risposta HTTP + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_TYPE, contentType) // Tipo MIME dinamico + .body(resource); + } catch (Exception e) { + return ResponseEntity.internalServerError().build(); + } +} +} diff --git a/src/main/java/com/olympus/apollo/services/ProjectService.java b/src/main/java/com/olympus/apollo/services/ProjectService.java new file mode 100644 index 0000000..15ff917 --- /dev/null +++ b/src/main/java/com/olympus/apollo/services/ProjectService.java @@ -0,0 +1,80 @@ +package com.olympus.apollo.services; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.olympus.apollo.repository.ProjectRepository; +import com.olympus.apollo.security.entity.User; +import com.olympus.apollo.security.repository.UserRepository; +import com.olympus.model.Application; +import com.olympus.model.Project; + +@Service +public class ProjectService { + + private Logger logger = LoggerFactory.getLogger(ProjectService.class); + + @Autowired + private ProjectRepository projectRepo; + + @Autowired + private UserRepository userRepo; + + public List getListProjectsByUser(){ + logger.info("getListProjectByUser function:"); + + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + return principal.getLstProjects(); + + } + + public User updateUserSelectedProject(Project idProject){ + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + principal.setSelectedProject(idProject); + principal.setSelectedApplication(null); + User u = userRepo.save(principal); + return u; + + } + + public User updateUserSelectedApplication(Application application){ + User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); + + principal.setSelectedApplication(application); + User u = userRepo.save(principal); + return u; + + } + + + public static List convertToObjectIdList(List listProjectIds) { + List objectIdList = new ArrayList<>(); + + for (String idString : listProjectIds) { + try { + // Crea un nuovo ObjectId dalla stringa e aggiungilo alla lista + objectIdList.add(new ObjectId(idString)); + } catch (IllegalArgumentException e) { + // Gestione degli errori per stringhe non valide che non possono essere convertite in ObjectId + System.err.println("Invalid ObjectId string: " + idString); + } + } + + return objectIdList; + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 9bbf31d..9e333ea 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -12,8 +12,8 @@ spring: ai: azure: openai: - endpoint: "https://ai-olympus.openai.azure.com/" - api-key: "9fb33cc69d914d4c8225b974876510b5" + endpoint: "https://ai-olympus-new.openai.azure.com/" + api-key: "4eHwvw6h7vHxTmI2870cR3EpEBs5L9sXZabr9nz37y39TXtk0xY5JQQJ99AKAC5RqLJXJ3w3AAABACOGLdow" openai: api-key: "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" vectorstore: @@ -23,10 +23,14 @@ spring: initialize-schema: true data: mongodb: - uri: mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus + uri: mongodb+srv://olympusadmin:Camilla123!@db-olympus.global.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000 database: olympus - username: olympus_adm - password: 26111979 + username: olympusadmin + password: Camilla123! + # uri: mongodb+srv://olympus_adm:26111979@olympus.l6qor4p.mongodb.net/?retryWrites=true&w=majority&appName=Olympus + # database: olympus + # username: olympus_adm + # password: 26111979 servlet: multipart: max-file-size: 5000000MB