diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 101aa2f..509e3f5 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -19,6 +19,6 @@ jobs: registry_username: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_USERNAME }} registry_token: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_TOKEN }} dockerfile: Dockerfile.daily-tests - tag: "0.6.3" + tag: "0.6.4" image_name: "upstream-daily-tests" quay_application_token: ${{ secrets.QUAY_IMAGE_SCLORG_UPDATE_DESC }} diff --git a/Dockerfile.daily-tests b/Dockerfile.daily-tests index 9cb294d..70a658e 100644 --- a/Dockerfile.daily-tests +++ b/Dockerfile.daily-tests @@ -2,7 +2,7 @@ FROM quay.io/fedora/fedora:42 ENV SHARED_DIR="/var/ci-scripts" \ VERSION="42" \ - RELEASE_UPSTREAM="0.6.3" \ + RELEASE_UPSTREAM="0.6.4" \ UPSTREAM_TMT_REPO="https://github.com/sclorg/sclorg-testing-farm" \ UPSTREAM_TMT_DIR="sclorg-testing-farm" \ HOME="/home/nightly" \ diff --git a/Makefile b/Makefile index dcc5f13..723b6dc 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ shellcheck: ./run-shellcheck.sh `git ls-files *.sh` build_images: - podman build -t quay.io/sclorg/upstream-daily-tests:0.6.3 -f Dockerfile.daily-tests . + podman build -t quay.io/sclorg/upstream-daily-tests:0.6.4 -f Dockerfile.daily-tests . diff --git a/daily_tests/daily_nightly_tests_report.py b/daily_tests/daily_nightly_tests_report.py index 17d313a..cb27eb4 100755 --- a/daily_tests/daily_nightly_tests_report.py +++ b/daily_tests/daily_nightly_tests_report.py @@ -64,75 +64,43 @@ TEST_CASES = { # Format is test for OS and king of test, what TMT Plan is used and MSG to mail - ("fedora-test", "nightly-container-f", "Fedora test results:"), - ("fedora-test-pytest", "nightly-container-f", "Fedora PyTest test results:"), - ("c9s-test", "nightly-container-centos-stream-9", "CentOS Stream 9 test results:"), - ( - "c9s-test-pytest", - "nightly-container-centos-stream-9", - "CentOS Stream 9 PyTest test results:", - ), - ( - "c10s-test", - "nightly-container-centos-stream-10", - "CentOS Stream 10 test results:", - ), - ( - "c10s-test-pytest", - "nightly-container-centos-stream-10", - "CentOS Stream 10 PyTest test results:", - ), - ("rhel8-test", "nightly-container-rhel8", "RHEL-8 test results:"), - ("rhel8-test-pytest", "nightly-container-rhel8", "RHEL-8 PyTest test results:"), - ( - "rhel8-test-openshift-4", - "nightly-container-rhel8", - "RHEL-8 OpenShift 4 test results:", - ), + ("fedora-test", "nightly-fedora", "Fedora test results:"), + ("fedora-test-pytest", "nightly-fedora", "Fedora PyTest test results:"), + ("c9s-test", "nightly-c9s", "CentOS Stream 9 test results:"), + ("c9s-test-pytest", "nightly-c9s", "CentOS Stream 9 PyTest test results:"), + ("c10s-test", "nightly-c10s", "CentOS Stream 10 test results:"), + ("c10s-test-pytest", "nightly-c10s", "CentOS Stream 10 PyTest test results:"), + ("rhel8-test", "nightly-rhel8", "RHEL-8 test results:"), + ("rhel8-test-pytest", "nightly-rhel8", "RHEL-8 PyTest test results:"), + ("rhel8-test-openshift-4", "nightly-rhel8", "RHEL-8 OpenShift 4 test results:"), ( "rhel8-test-openshift-pytest", - "nightly-container-rhel8", + "nightly-rhel8", "RHEL-8 PyTest in OpenShift 4 test results:", ), - ("rhel9-test", "nightly-container-rhel9", "RHEL-9 test results:"), - ("rhel9-test-pytest", "nightly-container-rhel8", "RHEL-9 PyTest test results:"), - ( - "rhel9-test-openshift-4", - "nightly-container-rhel9", - "RHEL-9 OpenShift 4 test results:", - ), + ("rhel9-test", "nightly-rhel9", "RHEL-9 test results:"), + ("rhel9-test-pytest", "nightly-rhel9", "RHEL-9 PyTest test results:"), + ("rhel9-test-openshift-4", "nightly-rhel9", "RHEL-9 OpenShift 4 test results:"), ( "rhel9-test-openshift-pytest", - "nightly-container-rhel9", + "nightly-rhel9", "RHEL-9 PyTest in OpenShift 4 test results:", ), - ( - "rhel9-helm-charts", - "nightly-container-rhel9", - "RHEL-9 Helm Charts test results:", - ), - ("rhel10-test", "nightly-container-rhel10", "RHEL-10 test results:"), - ("rhel10-test-pytest", "nightly-container-rhel10", "RHEL-10 PyTest test results:"), - ( - "rhel10-test-openshift-4", - "nightly-container-rhel10", - "RHEL-10 OpenShift 4 test results:", - ), + ("rhel9-helm-charts", "nightly-rhel9", "RHEL-9 Helm Charts test results:"), + ("rhel10-test", "nightly-rhel10", "RHEL-10 test results:"), + ("rhel10-test-pytest", "nightly-rhel10", "RHEL-10 PyTest test results:"), + ("rhel10-test-openshift-4", "nightly-rhel10", "RHEL-10 OpenShift 4 test results:"), ( "rhel10-test-openshift-pytest", - "nightly-container-rhel10", + "nightly-rhel10", "RHEL-10 PyTest in OpenShift 4 test results:", ), } TEST_UPSTREAM_CASES = { - ("rhel8-test-upstream", "nightly-container-rhel8", "RHEL-8 Upstream test results:"), - ("rhel9-test-upstream", "nightly-container-rhel9", "RHEL-9 Upstream test results:"), - ( - "rhel10-test-upstream", - "nightly-container-rhel10", - "RHEL-10 Upstream test results:", - ), + ("rhel8-test-upstream", "nightly-rhel8", "RHEL-8 Upstream test results:"), + ("rhel9-test-upstream", "nightly-rhel9", "RHEL-9 Upstream test results:"), + ("rhel10-test-upstream", "nightly-rhel10", "RHEL-10 Upstream test results:"), } # The default directory used for nightly build @@ -273,7 +241,7 @@ def collect_data(self): self.data_dict["SUCCESS_DATA"] = [] failed_tests = False for test_case, plan, _ in self.available_test_case: - path_dir = Path(RESULTS_DIR) / test_case + path_dir = Path(RESULTS_DIR) / test_case / "nightly" if not path_dir.is_dir(): print(f"The test case {path_dir} does not exists that is weird") continue diff --git a/run_in_container.sh b/run_in_container.sh new file mode 100755 index 0000000..d004083 --- /dev/null +++ b/run_in_container.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -x + +TARGET=$1 +TESTS=$2 +TEST_TYPE=$3 + +podman run --rm -it \ + -v /var/tmp/:/var/tmp/:Z \ + -e SHARED_DIR=/var/tmp/ \ + -e TARGET=$TARGET \ + -e TESTS=$TESTS \ + -e TEST_TYPE=$TEST_TYPE \ + quay.io/sclorg/daily-tests:latest diff --git a/run_nightly_tests.sh b/run_nightly_tests.sh index 31d28b1..ba5f0d0 100755 --- a/run_nightly_tests.sh +++ b/run_nightly_tests.sh @@ -2,27 +2,45 @@ set -x -[[ -z "$TARGET" ]] && { echo "You have to specify target to build SCL images. rhel9, rhel8, or fedora" && exit 1 ; } -[[ -z "$TESTS" ]] && { echo "You have to specify type of the test to run. test, test-pytest, test-openshift, test-openshift-pytest" && exit 1 ; } +if [[ -n "$TARGET" ]]; then + echo "Target is set to $TARGET" +else + echo "TARGET variable is not set. Please set it to rhel9, rhel8, rhel10, fedora, c9s or c10s." + exit 1 +fi +if [[ -n "$TESTS" ]]; then + echo "Test is set to $TESTS" +else + echo "TESTS variable is not set. Please set it to test, test-pytest, test-openshift, or test-openshift-pytest." + exit 1 +fi + SET_TEST="" if [[ "${TESTS}" != "test-upstream" ]]; then - [[ -z "$TEST_TYPE" ]] && { echo "You have to specify type of images S2I or NOS2I" && exit 1 ; } + if [[ -n "$TEST_TYPE" ]]; then + echo "Test type is set to $TEST_TYPE" + else + echo "TEST_TYPE variable is not set. Please set it to S2I or NOS2I." + exit 1 + fi SET_TEST="$TEST_TYPE" fi # Local working directories +CUR_DATE=$(date +%Y-%m-%d) WORK_DIR="${HOME}/ci-scripts/" LOCAL_LOGS_DIR="${HOME}/logs/" # Shared directories between runs -DAILY_REPORTS_DIR="${SHARED_DIR}/daily_reports_dir" -TFT_PLAN="nightly-container-$TARGET" -DAILY_REPORTS_TESTS_DIR="${DAILY_REPORTS_DIR}/${TARGET}-${TESTS}" -DAILY_SCLORG_TESTS_DIR="${SHARED_DIR}/daily_scl_tests" +DAILY_REPORTS_DIR="${SHARED_DIR}/daily_reports_dir/${CUR_DATE}" +TFT_PLAN="nightly/nightly-$TARGET" +DAILY_REPORTS_TESTS_DIR="${DAILY_REPORTS_DIR}/${CUR_DATE}/${TARGET}-${TESTS}" +DAILY_SCLORG_TESTS_DIR="${SHARED_DIR}/daily_scl_tests/${CUR_DATE}/${TARGET}-${TESTS}-${SET_TEST}" -DIR="${WORK_DIR}/${TARGET}-${TESTS}-${SET_TEST}" +DIR="${WORK_DIR}/${CUR_DATE}/${TARGET}-${TESTS}-${SET_TEST}" if [[ "$TESTS" == "test-upstream" ]]; then - DIR="${WORK_DIR}/${TARGET}-${TESTS}" + DIR="${WORK_DIR}/${CUR_DATE}/${TARGET}-${TESTS}" + DAILY_SCLORG_TESTS_DIR="${SHARED_DIR}/daily_scl_tests/${CUR_DATE}/${TARGET}-${TESTS}" fi LOG_FILE="${LOCAL_LOGS_DIR}/${TARGET}-${TESTS}.log" @@ -38,12 +56,6 @@ function generate_passwd_file() { export NSS_WRAPPER_GROUP=/etc/group } -function move_logs_to_old() { - echo "Moving logs to old directory" - mv "${DAILY_REPORTS_DIR}/*" "${DAILY_REPORTS_DIR}_old/" - echo "Logs moved to old directory" -} - function prepare_environment() { mkdir -p "${LOCAL_LOGS_DIR}" mkdir -p "${WORK_DIR}" @@ -65,15 +77,15 @@ function get_compose() { elif [[ "$TARGET" == "fedora" ]]; then COMPOSE="1MT-Fedora-${VERSION}" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" - TFT_PLAN="nightly-container-fedora" + TFT_PLAN="nightly/nightly-fedora" elif [[ "$TARGET" == "c9s" ]]; then COMPOSE="1MT-CentOS-Stream-9" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" - TFT_PLAN="nightly-container-c9s" + TFT_PLAN="nightly/nightly-c9s" elif [[ "$TARGET" == "c10s" ]]; then COMPOSE="1MT-CentOS-Stream-10" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" - TFT_PLAN="nightly-container-c10s" + TFT_PLAN="nightly/nightly-c10s" else echo "This target is not supported" exit 1 @@ -91,31 +103,26 @@ function run_tests() { fi TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision -v -v --how minute --auto-select-network --image ${COMPOSE}" echo "TMT command is: $TMT_COMMAND" | tee -a "${LOG_FILE}" - touch "${RESULTS_TARGET_DIR}/tmt_running" + touch "${DAILY_SCLORG_TESTS_DIR}/tmt_running" set -o pipefail $TMT_COMMAND | tee -a "${LOG_FILE}" - if [[ $? -ne 0 ]]; then + ret_code=$? + set +o pipefail + rm -f "${DAILY_SCLORG_TESTS_DIR}/tmt_running" + if [[ $ret_code -ne 0 ]]; then echo "TMT command $TMT_COMMAND has failed." - if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success" ]]; then - rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success" - fi touch "${DAILY_REPORTS_TESTS_DIR}/tmt_failed" else - if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_failed" ]]; then - echo "Previous test run has failed but this one has passed." - else - touch "${DAILY_REPORTS_TESTS_DIR}/tmt_success" - fi + touch "${DAILY_REPORTS_TESTS_DIR}/tmt_success" fi - ls -laR "${DIR}/plans/${TFT_PLAN}/data/" > "$DAILY_SCLORG_TESTS_DIR/all_files_${TARGET}_${TESTS}.txt" cp "${LOG_FILE}" "${DAILY_SCLORG_TESTS_DIR}/log_${TARGET}_${TESTS}.txt" if [[ -d "${DIR}/plans/${TFT_PLAN}/data" ]]; then + ls -laR "${DIR}/plans/${TFT_PLAN}/data/" > "$DAILY_SCLORG_TESTS_DIR/all_files_${TARGET}_${TESTS}.txt" + ls -la "${DIR}/plans/${TFT_PLAN}/data/results/" cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" fi cp "${DIR}/log.txt" "${DAILY_REPORTS_TESTS_DIR}/" - set +o pipefail - rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_running" } if [[ "$TESTS" != "test" ]] && [[ "$TESTS" != "test-pytest" ]] && [[ "$TESTS" != "test-upstream" ]] && [[ "$TESTS" != "test-openshift-pytest" ]] && [[ "$TESTS" != "test-openshift-4" ]]; then @@ -130,10 +137,11 @@ generate_passwd_file # chown -R "${USER_ID}":0 $HOME/ # chown -R "${USER_ID}":0 $WORK_DIR/ +move_logs_to_old + prepare_environment get_compose -# move_logs_to_old date > "${LOG_FILE}" curl --insecure -L https://url.corp.redhat.com/fmf-data > "/tmp/fmf_data" diff --git a/show_logs.py b/show_logs.py old mode 100644 new mode 100755 index 9f48367..eb3dad0 --- a/show_logs.py +++ b/show_logs.py @@ -2,48 +2,106 @@ import os import sys +from datetime import date from pathlib import Path DAILY_REPORTS_DIR = Path("/var/ci-scripts/daily_reports_dir/") DAILY_SCL_TESTS_DIR = Path("/var/ci-scripts/daily_scl_tests/") +TEST_CASES = { + # Format is test for OS and king of test, what TMT Plan is used and MSG to mail + ("c10s", "nightly/nightly-c10s"), + ("fedora", "nightly/nightly-fedora"), + ("c9s", "nightly/nightly-c9s"), + ("rhel8", "nightly/nightly-rhel8"), + ("rhel9", "nightly/nightly-rhel9"), + ("rhel10", "nightly/nightly-rhel10"), +} + class PVCWatcherReport: def __init__(self): self.cwd = os.getcwd() + self.date = date.today().strftime("%Y-%m-%d") + self.reports_dir = DAILY_REPORTS_DIR / self.date + self.scl_tests_dir = DAILY_SCL_TESTS_DIR / self.date def iter_results_in_directory(self): - success_tests = [] - failed_tests = [] - for item in DAILY_REPORTS_DIR.iterdir(): - print(f"Inspecting item: {item}") + success_tmt_plans = [] + failed_tmt_plans = [] + running_tmt_plans = [] + failed_container_tests = [] + print("Show status of all TMT/FMF plans:") + for item in self.scl_tests_dir.iterdir(): + if item.is_dir(): + if (item / "tmt_running").exists(): + running_tmt_plans.append(item.name) + for item in self.reports_dir.iterdir(): if item.is_dir(): if (item / "tmt_success").exists(): - success_tests.append(item.name) + success_tmt_plans.append(item.name) else: - failed_tests.append(item.name) - print(f"Success tests in {DAILY_REPORTS_DIR} are:") - print("\n".join(success_tests)) - print(f"Failed tests in {DAILY_REPORTS_DIR} are:") - print("\n".join(failed_tests)) + failed_tmt_plans.append(item.name) + for item in self.reports_dir.iterdir(): + if item.is_dir(): + failed_container_tests.extend( + self.return_failed_tests(self.reports_dir, item) + ) + if running_tmt_plans: + print("Running TMT plans that are not finished yet:") + print("\n".join(running_tmt_plans)) + if success_tmt_plans: + print(f"Success TMT plans in {self.reports_dir} are:") + print("\n".join(success_tmt_plans)) + if failed_tmt_plans: + print(f"Failed TMT plans in {self.reports_dir} are:") + print("\n".join(failed_tmt_plans)) + if failed_container_tests: + print(f"!!!!Failed container tests are: {failed_container_tests}!!!!") + + def return_failed_tests(self, directory, item) -> list: + plan_name = "".join([x[1] for x in TEST_CASES if item.name.startswith(x[0])]) + dir_path = directory / f"{item.name}/plans/{plan_name}/data/results" + print(f"Looking for failed tests in directory: {dir_path}") + return list(dir_path.rglob("*.log")) def iter_over_executed_tests(self): - """Yield all executed tests in the given directory.""" - executed_tests = [] - for item in DAILY_SCL_TESTS_DIR.iterdir(): - print(f"Inspecting item: {item}") - if item.is_file(): - executed_tests.append(item.name) - print("Executed tests are:") - print("\n".join(executed_tests)) + """View all executed tests in the given directory.""" + for item in self.scl_tests_dir.iterdir(): + print(f"Inspecting item in '{self.scl_tests_dir}' directory: {item}") + if not item.is_dir(): + continue + failed_container_tests = self.return_failed_tests(self.scl_tests_dir, item) + if not failed_container_tests: + print(f"No container test failures found in {item}.") + continue + print(f"!!!!Failed container tests are: {failed_container_tests}!!!!") + + def show_all_available_tests(self): + print("All previous available tests are:") + for item in DAILY_REPORTS_DIR.iterdir(): + if item.is_dir(): + print(item.name) def print_report(self): - print("Daily SCL Tests Reports:") + if not self.scl_tests_dir.is_dir(): + print( + f"The directory {self.scl_tests_dir} does not exist. Tests were not executed yet." + ) + return + print(f"Summary ({self.date}) of Daily SCL Tests Reports:") + if not self.reports_dir.is_dir(): + print( + f"The directory {self.reports_dir} does not exist. Tests were not finished yet." + ) + return self.iter_over_executed_tests() + print(f"Summary of results reports directory {self.reports_dir}:") self.iter_results_in_directory() if __name__ == "__main__": dgr = PVCWatcherReport() + dgr.show_all_available_tests() dgr.print_report() sys.exit(0) diff --git a/show_logs.sh b/show_logs.sh index 21234e7..be8ddc8 100755 --- a/show_logs.sh +++ b/show_logs.sh @@ -4,6 +4,7 @@ set -x while true; do echo "Displaying log files every 10 minutes..." + date python3 /root/ci-scripts/show_logs.py # Let's sleep for 10 minutes sleep 600