Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: update integration and evaluation tests pipelines #356

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

chore: update integration and evaluation tests pipelines #356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f5eb60f
update integration and evaluation tests pipelines
mfaizanse Jan 23, 2025
3bd6e3b
updated
mfaizanse Jan 23, 2025
300ebba
updated
mfaizanse Jan 23, 2025
4afbdeb
updated
mfaizanse Jan 23, 2025
5091317
updated
mfaizanse Jan 23, 2025
d9b9422
updated
mfaizanse Jan 23, 2025
545d29c
updated
mfaizanse Jan 23, 2025
17c6b63
updated
mfaizanse Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 19 additions & 60 deletions .github/workflows/pull-evaluation-tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
name: "Evaluation tests"
run-name: "Evaluation tests"

# Required repository secrets:
# - AICORE_SECRET
# - COMPANION_CONFIGMAP
# - GARDENER_TEST_CLUSTER_URL
# - GARDENER_TEST_CLUSTER_AUTH_TOKEN
# - GARDENER_TEST_CLUSTER_CA_DATA

on:
pull_request_target:
types: [labeled, opened, synchronize, reopened, ready_for_review]
Expand Down Expand Up @@ -80,6 +73,10 @@ jobs:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}

- name: Add config.json for companion
run: |
echo "${{ secrets.EVALUATION_TESTS_CONFIG }}" | base64 --decode | jq > $GITHUB_WORKSPACE/config/config.json

- name: K3d Setup - Install Kubectl CLI
run: |
curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
Expand Down Expand Up @@ -123,34 +120,25 @@ jobs:
echo "Installing Redis using Helm..."
helm install --wait redis-stack redis-stack/redis-stack --set auth.enabled=false -n redis

- name: Companion Deploy - Create namespace
run: |
kubectl apply -f scripts/k8s/companion-namespace.yaml

- name: Companion Deploy - Create secret
env:
COMPANION_CONFIG_BASE64: ${{ secrets.EVALUATION_TESTS_CONFIG }}
run: |
echo "${{ secrets.AICORE_SECRET }}" | base64 -d > scripts/k8s/aicore-secret.yaml
kubectl apply -f scripts/k8s/aicore-secret.yaml
rm -f scripts/k8s/aicore-secret.yaml
kubectl create namespace ai-system
./scripts/k8s/create-secret.sh

- name: Companion Deploy - Create ConfigMap
- name: Companion Deploy - Apply companion manifests
run: |
echo "${{ secrets.COMPANION_CONFIGMAP_JSON }}" | base64 -d > scripts/k8s/companion-configmap.yaml
kubectl apply -f scripts/k8s/companion-configmap.yaml
rm -f scripts/k8s/companion-configmap.yaml
kubectl apply -f scripts/k8s/companion-k3d-manifest.yaml

- name: Companion Deploy - Update image name in Deployment
run: |
sed -i "s|example.com/ai-backend:stable|${{env.IMAGE_NAME}}|g" scripts/k8s/companion-deployment.yaml

- name: Companion Deploy - Create Deployment and NodePort Service
run: |
kubectl apply -f scripts/k8s/companion-deployment.yaml
kubectl set image deployment/companion companion=${{env.IMAGE_NAME}}

- name: Companion Deploy - Wait for deployment
run: |
echo "Waiting for backend pods (labels: app=ai-backend) to be ready..."
kubectl wait --for=condition=Ready pod -l 'app=ai-backend' -n ai-core --timeout=300s
echo "Waiting for companion pods to be ready..."
kubectl wait --for=condition=Ready pod -l 'app.kubernetes.io/name=companion' -n ai-system --timeout=300s
sleep 30

- name: Companion Deploy - Test reachability through NodePort
Expand All @@ -160,9 +148,9 @@ jobs:
- name: Companion Deploy - Debug information
if: failure()
run: |
kubectl get deploy -n ai-core
kubectl get pod -n ai-core
kubectl logs -n ai-core -l app=ai-backend
kubectl get deploy -n ai-system
kubectl get pod -n ai-system
kubectl logs -n ai-system -l app.kubernetes.io/name=companion

- name: Evaluation Tests Setup - Extract project Python version
working-directory: tests/blackbox
Expand All @@ -185,47 +173,18 @@ jobs:
working-directory: tests/blackbox
run: poetry install

- name: Evaluation Tests Setup - Prepare test configurations
id: eval-tests-configs
env:
AICORE_SECRET: ${{ secrets.AICORE_SECRET }}
run: |
# AI Core configurations.
echo "${AICORE_SECRET}" | base64 -d > aicore-secret.yaml
echo "AICORE_CLIENT_ID=$(yq eval '.stringData.clientid' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_CLIENT_SECRET=$(yq eval '.stringData.clientsecret' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_AUTH_URL=$(yq eval '.stringData.auth_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_BASE_URL=$(yq eval '.stringData.base_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_RESOURCE_GROUP=$(yq eval '.stringData.resource_group' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_CONFIGURATION_ID_GPT4_MINI=$(yq eval '.stringData.configuration_id_gpt4_mini' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_DEPLOYMENT_ID_GPT4_MINI=$(yq eval '.stringData.deployment_id_gpt4_mini' aicore-secret.yaml)" >> $GITHUB_OUTPUT
# cleanup
rm -f aicore-secret.yaml

- name: Run Evaluation Tests
working-directory: tests/blackbox
env:
LOG_LEVEL: "DEBUG"
TEST_DATA_PATH: "./data"
COMPANION_API_URL: "http://localhost:32000"
# Gardener test cluster configurations.
TEST_CLUSTER_URL: ${{ secrets.GARDENER_TEST_CLUSTER_URL }}
TEST_CLUSTER_CA_DATA: ${{ secrets.GARDENER_TEST_CLUSTER_CA_DATA }}
TEST_CLUSTER_AUTH_TOKEN: ${{ secrets.GARDENER_TEST_CLUSTER_AUTH_TOKEN }}
# AI Core configurations.
AICORE_CLIENT_ID: ${{ steps.eval-tests-configs.outputs.AICORE_CLIENT_ID }}
AICORE_CLIENT_SECRET: ${{ steps.eval-tests-configs.outputs.AICORE_CLIENT_SECRET }}
AICORE_AUTH_URL: ${{ steps.eval-tests-configs.outputs.AICORE_AUTH_URL }}
AICORE_BASE_URL: ${{ steps.eval-tests-configs.outputs.AICORE_BASE_URL }}
AICORE_RESOURCE_GROUP: ${{ steps.eval-tests-configs.outputs.AICORE_RESOURCE_GROUP }}
AICORE_CONFIGURATION_ID_GPT4_MINI: ${{ steps.eval-tests-configs.outputs.AICORE_CONFIGURATION_ID_GPT4_MINI }}
AICORE_DEPLOYMENT_ID_GPT4_MINI: ${{ steps.eval-tests-configs.outputs.AICORE_DEPLOYMENT_ID_GPT4_MINI }}
run: |
poetry run python src/run_evaluation.py

- name: Evaluation Tests - Debug information
if: failure()
run: |
kubectl get deploy -n ai-core
kubectl get pod -n ai-core
kubectl logs -n ai-core -l app=ai-backend
kubectl get deploy -n ai-system
kubectl get pod -n ai-system
kubectl logs -n ai-system -l app.kubernetes.io/name=companion
40 changes: 0 additions & 40 deletions .github/workflows/pull-integration-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,49 +50,9 @@ jobs:
- name: Install dependencies
run: poetry install --with dev

- name: Prepare test configurations
id: int-tests-configs
env:
AICORE_SECRET: ${{ secrets.AICORE_SECRET }}
HANA_DB_SECRET: ${{ secrets.HANA_DB }}
run: |
echo "${AICORE_SECRET}" | base64 -d > aicore-secret.yaml
echo "AICORE_CLIENT_ID=$(yq eval '.stringData.clientid' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_CLIENT_SECRET=$(yq eval '.stringData.clientsecret' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_AUTH_URL=$(yq eval '.stringData.auth_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_BASE_URL=$(yq eval '.stringData.base_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT
echo "AICORE_RESOURCE_GROUP=$(yq eval '.stringData.resource_group' aicore-secret.yaml)" >> $GITHUB_OUTPUT

# Extract HANA DB credentials - updated key names
echo "${HANA_DB_SECRET}" > hana-db-secret.yaml
echo "DATABASE_URL=$(yq eval '.DATABASE_URL' hana-db-secret.yaml)" >> $GITHUB_OUTPUT
echo "DATABASE_USER=$(yq eval '.DATABASE_USER' hana-db-secret.yaml)" >> $GITHUB_OUTPUT
echo "DATABASE_PASSWORD=$(yq eval '.DATABASE_PASSWORD' hana-db-secret.yaml)" >> $GITHUB_OUTPUT
echo "DATABASE_PORT=$(yq eval '.DATABASE_PORT' hana-db-secret.yaml)" >> $GITHUB_OUTPUT
echo "DOCS_TABLE_NAME=$(yq eval '.DOCS_TABLE_NAME' hana-db-secret.yaml)" >> $GITHUB_OUTPUT

# cleanup
rm -f aicore-secret.yaml hana-db-secret.yaml

- name: Run integration tests
env:
LOG_LEVEL: "DEBUG"
# AI Core configurations.
AICORE_CLIENT_ID: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_ID }}
AICORE_CLIENT_SECRET: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_SECRET }}
AICORE_AUTH_URL: ${{ steps.int-tests-configs.outputs.AICORE_AUTH_URL }}
AICORE_BASE_URL: ${{ steps.int-tests-configs.outputs.AICORE_BASE_URL }}
AICORE_RESOURCE_GROUP: ${{ steps.int-tests-configs.outputs.AICORE_RESOURCE_GROUP }}
# Gardener test cluster configurations.
TEST_CLUSTER_URL: ${{ secrets.GARDENER_TEST_CLUSTER_URL }}
TEST_CLUSTER_CA_DATA: ${{ secrets.GARDENER_TEST_CLUSTER_CA_DATA }}
TEST_CLUSTER_AUTH_TOKEN: ${{ secrets.GARDENER_TEST_CLUSTER_AUTH_TOKEN }}
# HANA DB configurations
DATABASE_URL: ${{ steps.int-tests-configs.outputs.DATABASE_URL }}
DATABASE_USER: ${{ steps.int-tests-configs.outputs.DATABASE_USER }}
DATABASE_PASSWORD: ${{ steps.int-tests-configs.outputs.DATABASE_PASSWORD }}
DATABASE_PORT: ${{ steps.int-tests-configs.outputs.DATABASE_PORT }}
DOCS_TABLE_NAME: ${{ steps.int-tests-configs.outputs.DOCS_TABLE_NAME }}
run: |
echo "${{ secrets.CONFIG_TEST }}" | base64 --decode | jq > $GITHUB_WORKSPACE/config/config.json
poetry run poe test-integration
130 changes: 0 additions & 130 deletions scripts/k8s/companion-deployment.yaml
View file
Open in desktop

This file was deleted.

Loading
Loading