From 1c92fc6421547aada556d66bd1c5af3b75976ae3 Mon Sep 17 00:00:00 2001 From: Adolfo Aguirrezabal Date: Thu, 22 Aug 2024 14:11:54 +0200 Subject: [PATCH] [CI] Run tests from trustyai-tests (#279) * Change Dockerfile to clone trustyai-tests * Add PYTEST_MARKERS env and remove TESTS_REGEX --- tests/Dockerfile | 19 ++++++++----------- tests/Makefile | 10 ++++++---- tests/README.md | 6 +----- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index e39ebdfd..833e6989 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,11 +1,7 @@ FROM registry.access.redhat.com/ubi8:8.10-1020 - ARG ORG=trustyai-explainability ARG BRANCH=main ARG ODS_CI_REPO=https://github.com/red-hat-data-services/ods-ci -# This git reference should always reference a stable commit from ods-ci that supports ODH -# This hash corresponds to a March 24th, 2023 commit -ARG ODS_CI_GITREF=a8cf770b37caa4ef7ce6596acc8bdd6866cc7772 ARG OC_CLI_URL=https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/4.14.33/openshift-client-linux.tar.gz ENV HOME /root @@ -36,18 +32,19 @@ COPY Pipfile Pipfile.lock $HOME/peak/ ## Grab CI scripts from single-source-of-truth RUN mkdir -p $HOME/peak/operator-tests/trustyai-explainability/ &&\ mkdir $HOME/kfdef/ &&\ - cp -r $HOME/src/trustyai-explainability/tests/resources/ $HOME/peak/operator-tests/trustyai-explainability/resources &&\ - cp $HOME/src/trustyai-explainability/tests/util $HOME/peak/operator-tests/trustyai-explainability &&\ - cp -r $HOME/src/trustyai-explainability/tests/basictests $HOME/peak/operator-tests/trustyai-explainability/basictests &&\ cp -r $HOME/src/trustyai-explainability/tests/setup/odh-*.yaml $HOME/kfdef/ &&\ cp -r $HOME/src/trustyai-explainability/tests/setup/*setup $HOME/peak/ &&\ cp -r $HOME/src/trustyai-explainability/tests/scripts/installandtest.sh $HOME/peak/ +# Install poetry to support the exeuction of trustyai-tests +RUN curl -sSL https://install.python-poetry.org | python3 - +ENV PATH="${PATH}:$HOME/.local/bin" +RUN cd $HOME/peak && \ + git clone https://github.com/trustyai-explainability/trustyai-tests.git && \ + cd trustyai-tests && \ + poetry install + COPY scripts/install.sh $HOME/peak/ -#COPY resources $HOME/peak/operator-tests/trustyai-explainability/resources -#COPY util $HOME/peak/operator-tests/trustyai-explainability -#COPY setup/odh-core.yaml $HOME/kfdef/ -#COPY basictests $HOME/peak/operator-tests/trustyai-explainability/basictests RUN chmod -R 777 $HOME/kfdef && \ mkdir -p $HOME/.kube && \ diff --git a/tests/Makefile b/tests/Makefile index 8fa56988..a82e260d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,8 +15,8 @@ OPENSHIFT_TESTUSER_PASS= OPENSHIFT_TESTUSER_LOGIN_PROVIDER= # Setting SKIP_INSTALL will let you run the tests against an ODH instance that is already setup SKIP_INSTALL= -# Setting TESTS_REGEX will allow you to change which tests are going to be run -TESTS_REGEX= +# Pytest markers to select the tests that will be executed +PYTEST_MARKERS= # Location inside the container where CI system will retrieve files after a test run ARTIFACT_DIR=/tmp/artifacts LOCAL_ARTIFACT_DIR="${PWD}/artifacts" @@ -25,17 +25,19 @@ BUILD_TOOL?=podman NO_CACHE?=false LOCAL?=false TEARDOWN?=false +PLATFORM?=linux/amd64 + all: test test: build run clean build: - ${BUILD_TOOL} build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) --build-arg ODS_CI_REPO=$(ODS_CI_REPO) --build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) --build-arg OC_CLI_URL=$(OC_CLI_URL) . + ${BUILD_TOOL} build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) --build-arg ODS_CI_REPO=$(ODS_CI_REPO) --build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) --build-arg OC_CLI_URL=$(OC_CLI_URL) --platform=$(PLATFORM) . --progress=plain run: # Confirm that we have a directory for storing any screenshots from selenium tests mkdir -p ${LOCAL_ARTIFACT_DIR}/screenshots oc config view --flatten --minify > /tmp/tests-kubeconfig - ${BUILD_TOOL} run -e SKIP_INSTALL=$(SKIP_INSTALL) -e TESTS_REGEX=$(TESTS_REGEX) -e SKIP_OPERATOR_INSTALL=$(SKIP_OPERATOR_INSTALL) \ + ${BUILD_TOOL} run -e SKIP_INSTALL=$(SKIP_INSTALL) -e PYTEST_MARKERS=$(PYTEST_MARKERS) -e SKIP_OPERATOR_INSTALL=$(SKIP_OPERATOR_INSTALL) \ -e SKIP_KFDEF_INSTALL=$(SKIP_KFDEF_INSTALL) -e ODHPROJECT=$(ODHPROJECT) \ -e OPENSHIFT_TESTUSER_NAME="$(OPENSHIFT_TESTUSER_NAME)" -e OPENSHIFT_TESTUSER_PASS="$(OPENSHIFT_TESTUSER_PASS)" -e OPENSHIFT_TESTUSER_LOGIN_PROVIDER=$(OPENSHIFT_TESTUSER_LOGIN_PROVIDER) -e ARTIFACT_DIR=$(ARTIFACT_DIR) \ -e LOCAL=$(LOCAL) -e TEARDOWN=$(TEARDOWN)\ diff --git a/tests/README.md b/tests/README.md index 3dcbd3f3..4864e3aa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -24,7 +24,7 @@ make clean # remove the artifacts of the test from the cluster (operator, ODH, p * `BUILD_TOOL=docker/podman`: set the tool used to build and run the testing container * `SKIP_INSTALL=true/false`: skip the install of the ODH operator, if you've already installed it manually or via a previous test * `SKIP_KFDEF_INSTALL=true/false`: skip the install of ODH via KFdef, if you've already installed it manually or via a previous test -* `TESTS_REGEX=${REGEX}`: only run tests whose names match the regex +* `PYTEST_MARKERS`: Used to select the tests that will be executed. [Available markers](https://github.com/trustyai-explainability/trustyai-tests/blob/main/pyproject.toml). * `LOCAL=true/false`: This flag makes the test suite stop and wait for user input between the end of a test script and cluster teardown. This prevents automatic teardown, which is useful for manual inspection of the cluster before teardown when running the tests locally. * `TEARDOWN=true/false`: This flag will just run the corresponding `teardown` functions within the various tests, useful for cleaning up stranded components from failed tests, without deleting the operator and ODH install. It's recommended to use this with a `make run`, as using `make test` will trigger a `make clean` that fully wipes the cluster. @@ -47,10 +47,6 @@ If you'd like to run the tests against an instance that already has a KfDef crea you set `SKIP_KFDEF_INSTALL=true` and that will cause the test run to skip the step of creating the default KfDef. example: `make run SKIP_KFDEF_INSTALL=true` -If you'd like to run a single test instead of all tests, you can -set the TESTS_REGEX variable `TESTS_REGEX=`. That will -only run the test that you specify instead of all of the tests. example: `make run TESTS_REGEX=grafana` - If you have a local instance already running the operator and you'd like to skip that part of the install process, you can set `SKIP_OPERATOR_INSTALL=true` and that will bypass installation of the operator, but will still install the authentication for any user tests.