Update setup.py

.black.toml

@@ -16,6 +16,8 @@ exclude = '''
     | buck-out
     | build
     | dist
+    | alembic
+    | gen
   )/
 )
 '''

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2.1
 orbs:
   python: circleci/[email protected]
+  aws-cli: circleci/[email protected]
 
 workflows:
   ci:
@@ -10,11 +11,16 @@ workflows:
       - integration_tests
       - build_image
       - build_docs
+      - deploy_docs:
+          filters:
+            branches:
+              only:
+                - main
 
 jobs:
   run_unit_tests_python_client:
     docker:
-      - image: python:3.8-bookworm
+      - image: python:3.10-bookworm
     resource_class: small
     parallelism: 1
     steps:
@@ -28,7 +34,7 @@ jobs:
       - run_unit_tests_python_client
   run_unit_tests_server:
     docker:
-      - image: python:3.8-bookworm
+      - image: python:3.10-bookworm
         environment:
           ML_INFRA_DATABASE_URL: postgresql://postgres@localhost/circle_test
       - image: circleci/postgres:12.9-postgis-ram
@@ -48,7 +54,7 @@ jobs:
       - run_unit_tests_server
   build_docs:
     docker:
-      - image: python:3.8-bookworm
+      - image: python:3.10-bookworm
     resource_class: small
     parallelism: 1
     steps:
@@ -62,41 +68,131 @@ jobs:
           name: Build Docs
           command: |
             mkdocs build --strict
+  deploy_docs:
+    docker:
+      - image: python:3.10-bookworm
+    resource_class: small
+    parallelism: 1
+    steps:
+      - add_ssh_keys: # gives write access to CircleCI worker
+          fingerprints:
+            - "76:0c:1b:9e:e3:6a:c3:5c:6f:24:91:ef:7c:54:d2:7a"
+      - checkout # checkout source code to working directory
+      - environment_setup
+      - install_client
+      - python/install-packages:
+          pkg-manager: pip
+          pip-dependency-file: requirements-docs.txt
+      - run:
+          name: Deploy Docs
+          command: |
+            mkdocs gh-deploy
   build_image:
     executor: ubuntu-large
     steps:
       - checkout
       - run:
           name: Build Docker Image
           command: |
-            docker build . -f server/Dockerfile -t llm-engine:$CIRCLE_SHA1
+            docker build . -f model-engine/Dockerfile -t model-engine:$CIRCLE_SHA1
   integration_tests:
     executor: ubuntu-large
     steps:
       - checkout
+      - aws-cli/setup:
+          role-arn: ${CIRCLECI_ROLE_ARN}
+          aws-region: AWS_REGION
+      - run:
+          name: Build Docker Image
+          command: |
+            docker build . -f model-engine/Dockerfile -t model-engine:$CIRCLE_SHA1
       - run:
           name: Install minikube
           command: |
             cd $HOME
             curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
             sudo dpkg -i minikube_latest_amd64.deb
-            minikube start --vm-driver=docker --kubernetes-version=v1.23.0 --memory=14336 --cpus=8
+            minikube start --vm-driver=docker --kubernetes-version=v1.23.0 --memory=49152 --cpus=14
       - run:
-          name: Install helm
+          name: Install kubectl, helm
           command: |
-            cd $HOME
+            cd $HOME/bin
             curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+            curl -LO "https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl"
+            chmod +x kubectl
+      - run:
+          name: Install helm chart dependencies (Redis, Postgres, Istio)
+          command: |
+            sudo apt-get update && sudo apt-get install -y expect
+            pushd $HOME/project/.circleci/resources
+            kubectl create namespace model-engine
+            kubectl apply -f redis-k8s.yaml
+            kubectl apply -f postgres-k8s.yaml
+            kubectl create secret generic model-engine-postgres-credentials --from-literal=database_url=postgresql://postgres:[email protected]:5432/circle_test
+            kubectl create secret generic model-engine-postgres-credentials --from-literal=database_url=postgresql://postgres:[email protected]:5432/circle_test -n model-engine
+            export ISTIO_VERSION=1.15.0
+            popd
+            curl -L https://istio.io/downloadIstio | TARGET_ARCH=x86_64 sh -
+            install istio-${ISTIO_VERSION}/bin/istioctl $HOME/bin
+            $HOME/bin/istioctl install --set profile=demo -y
+            kubectl create configmap default-config --from-literal=config="$(cat $HOME/project/.circleci/resources/.minikube-config-map | envsubst)"
+            kubectl create configmap default-config --namespace model-engine --from-literal=config="$(cat $HOME/project/.circleci/resources/.minikube-config-map | envsubst)"
+            cat $HOME/project/.circleci/resources/.minikube-registry-creds | envsubst | expect
+            minikube addons enable registry-creds
+      - run:
+          name: Pre-load model-engine image to minikube
+          command: |
+            minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1
+      - run:
+          name: Pre-load integration test images to minikube
+          command: |
+            docker build -f model-engine/model_engine_server/inference/pytorch_or_tf.base.Dockerfile \
+            --build-arg BASE_IMAGE=python:3.8-slim \
+            --build-arg REQUIREMENTS_FILE="$CIRCLE_SHA1-base-requirements.txt" \
+            -t temp:1.11.0-cuda11.3-cudnn8-runtime-$CIRCLE_SHA1 .
+
+            touch $CIRCLE_SHA1-requirements.txt
+            echo -e "cloudpickle==2.1.0\npyyaml==6.0" > $CIRCLE_SHA1-requirements.txt
+
+            DOCKER_BUILDKIT=1 docker build -f model-engine/model_engine_server/inference/pytorch_or_tf.user.Dockerfile \
+            --build-arg BASE_IMAGE=temp:1.11.0-cuda11.3-cudnn8-runtime-$CIRCLE_SHA1 \
+            --build-arg REQUIREMENTS_FILE="$CIRCLE_SHA1-requirements.txt" \
+            -t $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/hosted-model-inference/async-pytorch:1.11.0-cuda11.3-cudnn8-runtime-$CIRCLE_SHA1-b8c25b .
+            rm $CIRCLE_SHA1-requirements.txt
+
+            minikube --logtostderr -v 1 image load $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/hosted-model-inference/async-pytorch:1.11.0-cuda11.3-cudnn8-runtime-$CIRCLE_SHA1-b8c25b
       - run:
           name: Install helm chart
           command: |
-            cd $HOME/project/charts
-            helm install llm-engine llm-engine --values llm-engine/values_sample.yaml
+            pushd $HOME/project/charts
+            cat model-engine/values_circleci.yaml | envsubst > model-engine/values_circleci_subst.yaml
+            helm install model-engine model-engine --values model-engine/values_circleci_subst.yaml --set tag=$CIRCLE_SHA1 --atomic --debug
+      - run:
+          name: Change python version to 3.10.14
+          command: |
+            pyenv install 3.10.14
+            pyenv global 3.10.14
+      - run:
+          name: Install integration test dependencies
+          command: |
+            export DEBIAN_FRONTEND=noninteractive
+            sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev python3-dev
+            pip install -r model-engine/requirements.txt
+      - install_client
+      - install_server
+      - run:
+          name: Run integration tests
+          command: |
+            pushd $HOME/project
+            kubectl port-forward svc/model-engine 5001:80 &
+            export GIT_TAG=$CIRCLE_SHA1
+            pytest integration_tests
 
 executors:
   ubuntu-large:
     machine:
-      image: "ubuntu-2004:202201-02"
-    resource_class: xlarge
+      image: default
+    resource_class: 2xlarge
 
 commands:
   environment_setup:
@@ -112,29 +208,30 @@ commands:
   install_server:
     description: Installs LLM Engine server
     steps:
-     - python/install-packages:
-         pkg-manager: pip
-         app-dir: server
-     - python/install-packages:
-         pkg-manager: pip
-         app-dir: server
-         pip-dependency-file: requirements-test.txt
-     - python/install-packages:
-         pkg-manager: pip
-         app-dir: server
-         pip-dependency-file: requirements_override.txt
-     - run:
-         name: Install Server
-         command: |
-           pushd server
-           pip install -e .
-           popd
+      - python/install-packages:
+          pkg-manager: pip
+          app-dir: model-engine
+      - python/install-packages:
+          pkg-manager: pip
+          app-dir: model-engine
+          pip-dependency-file: requirements-test.txt
+      - python/install-packages:
+          pkg-manager: pip
+          app-dir: model-engine
+          pip-dependency-file: requirements_override.txt
+      - run:
+          name: Install Server
+          command: |
+            pushd model-engine
+            pip install -e .
+            popd
   install_client:
     description: Install LLM Engine client
     steps:
       - run:
           name: Install LLM Engine client
           command: |
+            pip install --upgrade pip
             pip install -e $HOME/project/clients/python
   run_unit_tests_python_client:
     description: Unit tests of the python client
@@ -159,16 +256,17 @@ commands:
       - run:
           name: Ruff Lint Check
           command: |
-            ruff .
+            ruff check .
       - run:
           name: Type Check
           command: |
-            pushd server
+            pushd model-engine
             mypy . --install-types --non-interactive
             popd
       - run:
           name: Unit Tests
           command: |
-            pushd server
-            WORKSPACE=.. pytest
+            pushd model-engine
+            GIT_TAG=$(git rev-parse HEAD) WORKSPACE=.. pytest --cov --cov-report=xml
+            diff-cover coverage.xml --compare-branch=origin/main --fail-under=80
             popd

.circleci/resources/.minikube-config-map

@@ -0,0 +1,5 @@
+# Configmap for AWS credentials inside minikube.
+[default]
+aws_access_key_id = $AWS_ACCESS_KEY_ID
+aws_secret_access_key = $AWS_SECRET_ACCESS_KEY
+aws_session_token = $AWS_SESSION_TOKEN

.circleci/resources/.minikube-registry-creds

@@ -0,0 +1,15 @@
+# Script to send the registry-creds addon configuration to minikube
+# Source: https://github.com/kubernetes/minikube/issues/8283
+# See expect syntax here: https://manpages.ubuntu.com/manpages/trusty/man1/expect.1.html
+spawn minikube addons configure registry-creds
+expect "Do you want to enable AWS Elastic Container Registry?" { send "y\r" }
+expect "Enter AWS Access Key ID:" { send "$AWS_ACCESS_KEY_ID\r" }
+expect "Enter AWS Secret Access Key:" { send "$AWS_SECRET_ACCESS_KEY\r" }
+expect "Enter AWS Session Token:" { send "$AWS_SESSION_TOKEN\r" }
+expect "Enter AWS Region:" { send "us-west-2\r" }
+expect "Enter 12 digit AWS Account ID (Comma separated list):" { send "$CIRCLECI_AWS_ACCOUNT_ID\r" }
+expect "Enter ARN of AWS role to assume:" { send "\r" }
+expect "Do you want to enable Google Container Registry?" { send "n\r" }
+expect "Do you want to enable Docker Registry?" { send "n\r" }
+expect "Do you want to enable Azure Container Registry?" { send "n\r" }
+expect eof

.circleci/resources/postgres-k8s.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+      - name: main
+        image: "cimg/postgres:12.8-postgis"
+        imagePullPolicy: IfNotPresent
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+        ports:
+        - containerPort: 5432
+        env:
+        - name: POSTGRES_USER
+          value: postgres
+        - name: POSTGRES_DB
+          value: circle_test
+        - name: POSTGRES_PASSWORD
+          value: circle_test
+
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: postgres
+  ports:
+    - name: redis
+      port: 5432
+      targetPort: 5432

.circleci/resources/redis-k8s.yaml

@@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-message-broker-master
+  labels:
+    app: redis-message-broker-master
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis-message-broker-master
+  template:
+    metadata:
+      labels:
+        app: redis-message-broker-master
+    spec:
+      containers:
+      - name: main
+        image: redis
+        imagePullPolicy: IfNotPresent
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+        ports:
+        - containerPort: 6379
+
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: redis-message-broker-master
+  labels:
+    app: redis-message-broker-master
+spec:
+  type: ClusterIP
+  selector:
+    app: redis-message-broker-master
+  ports:
+    - name: redis
+      port: 6379
+      targetPort: 6379