[CI] Run tests from trustyai-tests (#279)

* Change Dockerfile to clone trustyai-tests

* Add PYTEST_MARKERS env and remove TESTS_REGEX

tests/Dockerfile

@@ -1,11 +1,7 @@
 FROM registry.access.redhat.com/ubi8:8.10-1020
-
 ARG ORG=trustyai-explainability
 ARG BRANCH=main
 ARG ODS_CI_REPO=https://github.com/red-hat-data-services/ods-ci
-# This git reference should always reference a stable commit from ods-ci that supports ODH
-# This hash corresponds to a March 24th, 2023 commit
-ARG ODS_CI_GITREF=a8cf770b37caa4ef7ce6596acc8bdd6866cc7772
 ARG OC_CLI_URL=https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/4.14.33/openshift-client-linux.tar.gz
 
 ENV HOME /root
@@ -36,18 +32,19 @@ COPY Pipfile Pipfile.lock $HOME/peak/
 ## Grab CI scripts from single-source-of-truth
 RUN mkdir -p $HOME/peak/operator-tests/trustyai-explainability/ &&\
     mkdir $HOME/kfdef/ &&\
-    cp -r $HOME/src/trustyai-explainability/tests/resources/ $HOME/peak/operator-tests/trustyai-explainability/resources &&\
-    cp $HOME/src/trustyai-explainability/tests/util $HOME/peak/operator-tests/trustyai-explainability &&\
-    cp -r $HOME/src/trustyai-explainability/tests/basictests $HOME/peak/operator-tests/trustyai-explainability/basictests &&\
     cp -r $HOME/src/trustyai-explainability/tests/setup/odh-*.yaml $HOME/kfdef/ &&\
     cp -r $HOME/src/trustyai-explainability/tests/setup/*setup $HOME/peak/ &&\
     cp -r $HOME/src/trustyai-explainability/tests/scripts/installandtest.sh $HOME/peak/
 
+# Install poetry to support the exeuction of trustyai-tests
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="${PATH}:$HOME/.local/bin"
+RUN cd $HOME/peak && \
+    git clone https://github.com/trustyai-explainability/trustyai-tests.git && \
+    cd trustyai-tests && \
+    poetry install
+
 COPY scripts/install.sh $HOME/peak/
-#COPY resources $HOME/peak/operator-tests/trustyai-explainability/resources
-#COPY util $HOME/peak/operator-tests/trustyai-explainability
-#COPY setup/odh-core.yaml $HOME/kfdef/
-#COPY basictests $HOME/peak/operator-tests/trustyai-explainability/basictests
 
 RUN chmod -R 777 $HOME/kfdef && \
     mkdir -p $HOME/.kube && \

tests/Makefile

@@ -15,8 +15,8 @@ OPENSHIFT_TESTUSER_PASS=
 OPENSHIFT_TESTUSER_LOGIN_PROVIDER=
 # Setting SKIP_INSTALL will let you run the tests against an ODH instance that is already setup
 SKIP_INSTALL=
-# Setting TESTS_REGEX will allow you to change which tests are going to be run
-TESTS_REGEX=
+# Pytest markers to select the tests that will be executed
+PYTEST_MARKERS=
 # Location inside the container where CI system will retrieve files after a test run
 ARTIFACT_DIR=/tmp/artifacts
 LOCAL_ARTIFACT_DIR="${PWD}/artifacts"
@@ -25,17 +25,19 @@ BUILD_TOOL?=podman
 NO_CACHE?=false
 LOCAL?=false
 TEARDOWN?=false
+PLATFORM?=linux/amd64
+
 
 all: test
 test: build run clean
 
 build:
-	${BUILD_TOOL} build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) --build-arg ODS_CI_REPO=$(ODS_CI_REPO) --build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) --build-arg OC_CLI_URL=$(OC_CLI_URL) .
+	${BUILD_TOOL} build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) --build-arg ODS_CI_REPO=$(ODS_CI_REPO) --build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) --build-arg OC_CLI_URL=$(OC_CLI_URL) --platform=$(PLATFORM) .  --progress=plain
 run:
 	# Confirm that we have a directory for storing any screenshots from selenium tests
 	mkdir -p ${LOCAL_ARTIFACT_DIR}/screenshots
 	oc config view --flatten --minify > /tmp/tests-kubeconfig
-	${BUILD_TOOL} run -e SKIP_INSTALL=$(SKIP_INSTALL) -e TESTS_REGEX=$(TESTS_REGEX) -e SKIP_OPERATOR_INSTALL=$(SKIP_OPERATOR_INSTALL) \
+	${BUILD_TOOL} run -e SKIP_INSTALL=$(SKIP_INSTALL) -e PYTEST_MARKERS=$(PYTEST_MARKERS) -e SKIP_OPERATOR_INSTALL=$(SKIP_OPERATOR_INSTALL) \
 	    -e SKIP_KFDEF_INSTALL=$(SKIP_KFDEF_INSTALL) -e ODHPROJECT=$(ODHPROJECT) \
 		-e OPENSHIFT_TESTUSER_NAME="$(OPENSHIFT_TESTUSER_NAME)" -e OPENSHIFT_TESTUSER_PASS="$(OPENSHIFT_TESTUSER_PASS)" -e OPENSHIFT_TESTUSER_LOGIN_PROVIDER=$(OPENSHIFT_TESTUSER_LOGIN_PROVIDER) -e ARTIFACT_DIR=$(ARTIFACT_DIR) \
 		-e LOCAL=$(LOCAL) -e TEARDOWN=$(TEARDOWN)\

tests/README.md

@@ -24,7 +24,7 @@ make clean # remove the artifacts of the test from the cluster (operator, ODH, p
 * `BUILD_TOOL=docker/podman`: set the tool used to build and run the testing container
 * `SKIP_INSTALL=true/false`: skip the install of the ODH operator, if you've already installed it manually or via a previous test
 * `SKIP_KFDEF_INSTALL=true/false`: skip the install of ODH via KFdef, if you've already installed it manually or via a previous test
-* `TESTS_REGEX=${REGEX}`: only run tests whose names match the regex
+* `PYTEST_MARKERS`: Used to select the tests that will be executed. [Available markers](https://github.com/trustyai-explainability/trustyai-tests/blob/main/pyproject.toml).
 * `LOCAL=true/false`: This flag makes the test suite stop and wait for user input between the end of a test script and cluster teardown.   This prevents automatic teardown, which is useful for manual inspection of the cluster before teardown when running the tests locally.
 * `TEARDOWN=true/false`: This flag will just run the corresponding `teardown` functions within the various tests, useful for cleaning up stranded components from failed tests, without deleting the operator and ODH install. It's recommended to use this with a `make run`, as using `make test` will trigger a `make clean` that fully wipes the cluster. 
 
@@ -47,10 +47,6 @@ If you'd like to run the tests against an instance that already has a KfDef crea
 you set `SKIP_KFDEF_INSTALL=true` and that will cause the test run
 to skip the step of creating the default KfDef.  example: `make run SKIP_KFDEF_INSTALL=true`
 
-If you'd like to run a single test instead of all tests, you can
-set the TESTS_REGEX variable `TESTS_REGEX=<name of the test to run>`.  That will
-only run the test that you specify instead of all of the tests.  example: `make run TESTS_REGEX=grafana`
-
 If you have a local instance already running the operator and you'd like to skip that part
 of the install process, you can set `SKIP_OPERATOR_INSTALL=true` and that will bypass installation
 of the operator, but will still install the authentication for any user tests.