diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 59393b5..9fe66cc 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -19,6 +19,6 @@ jobs: registry_username: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_USERNAME }} registry_token: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_TOKEN }} dockerfile: Dockerfile.daily-tests - tag: "0.5.0" + tag: "0.6.0" image_name: "upstream-daily-tests" quay_application_token: ${{ secrets.QUAY_IMAGE_SCLORG_UPDATE_DESC }} diff --git a/Dockerfile.daily-tests b/Dockerfile.daily-tests index 9531523..bc5faab 100644 --- a/Dockerfile.daily-tests +++ b/Dockerfile.daily-tests @@ -1,15 +1,15 @@ FROM quay.io/fedora/fedora:42 -ENV WORK_DIR="/var/tmp" \ +ENV SHARED_DIR="/var/ci-scripts" \ VERSION="42" \ - RELEASE_UPSTREAM="0.5.0" \ - DOWNSTREAM_TMT_DIR="sclorg-tmt-plans" \ + RELEASE_UPSTREAM="0.6.0" \ UPSTREAM_TMT_REPO="https://github.com/sclorg/sclorg-testing-farm" \ UPSTREAM_TMT_DIR="sclorg-testing-farm" \ HOME="/home/nightly" \ SUMMARY="Daily tests for SCL org projects" \ DESCRIPTION="This image is used to run daily tests for SCL org projects in CI." \ - NAME="nightly" + NAME="nightly" \ + WORK_DIR="/home/nightly/ci-scripts" LABEL summary="${SUMMARY}" \ description="${DESCRIPTION}" \ @@ -18,15 +18,16 @@ LABEL summary="${SUMMARY}" \ io.k8s.vendor="SCL org" \ architecture="x86_64" -COPY requirements.sh "${WORK_DIR}/requirements.sh" -RUN bash "${WORK_DIR}/requirements.sh" - -RUN dnf install -y python3.13-pip git && \ +RUN dnf install -y python3.13-pip git nss_wrapper && \ dnf clean all -RUN git clone "${UPSTREAM_TMT_REPO}" "${WORK_DIR}/${UPSTREAM_TMT_DIR}" +COPY requirements.sh "${WORK_DIR}/requirements.sh" +RUN bash "${WORK_DIR}/requirements.sh" +RUN cd "${WORK_DIR}" && git clone "${UPSTREAM_TMT_REPO}" "${UPSTREAM_TMT_DIR}" COPY . /root/ci-scripts WORKDIR "${HOME}" +# USER 900 + CMD ["/root/ci-scripts/run_nightly_tests.sh"] diff --git a/Makefile b/Makefile index b5dcc51..7268d9f 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ shellcheck: ./run-shellcheck.sh `git ls-files *.sh` build_images: - podman build -t quay.io/sclorg/upstream-daily-tests:0.5.0 -f Dockerfile.daily-tests . + podman build -t quay.io/sclorg/upstream-daily-tests:0.6.0 -f Dockerfile.daily-tests . diff --git a/requirements.sh b/requirements.sh index 184a4ad..41e91b6 100755 --- a/requirements.sh +++ b/requirements.sh @@ -2,6 +2,6 @@ set -ex -useradd -u 500 -r -g 0 -m -s /bin/bash -c "Default Application User" "${NAME}" -chown -R 500:0 "${HOME}" +useradd -u 900 -r -g 0 -m -s /bin/bash -c "Default Application User" "${NAME}" +chown -R 900:0 "${HOME}" chmod -R a+rwx "${HOME}" diff --git a/run_nightly_tests.sh b/run_nightly_tests.sh index a971bd8..31180bf 100755 --- a/run_nightly_tests.sh +++ b/run_nightly_tests.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -x + [[ -z "$TARGET" ]] && { echo "You have to specify target to build SCL images. rhel9, rhel8, or fedora" && exit 1 ; } [[ -z "$TESTS" ]] && { echo "You have to specify type of the test to run. test, test-pytest, test-openshift, test-openshift-pytest" && exit 1 ; } SET_TEST="" @@ -8,48 +10,46 @@ if [[ "${TESTS}" != "test-upstream" ]]; then SET_TEST="$TEST_TYPE" fi -LOGS_DIR="${WORK_DIR}/daily_tests_logs/" -LOGS_DIR_OLD="${LOGS_DIR}/old" -DAILY_TEST_DIR="${WORK_DIR}/daily_scl_tests" -RESULTS_DIR="${WORK_DIR}/daily_reports_dir" -RESULTS_DIR_OLD="${RESULTS_DIR}/old" -SCRIPT="daily_scl_tests" +# Local working directories +WORK_DIR="${HOME}/ci-scripts/" +LOCAL_LOGS_DIR="${HOME}/logs/" + +# Shared directories between runs +DAILY_REPORTS_DIR="${SHARED_DIR}/daily_reports_dir" TFT_PLAN="nightly-container-$TARGET" -DIR="${DAILY_TEST_DIR}/${TARGET}-${TESTS}-${SET_TEST}" -RESULTS_TARGET_DIR="${RESULTS_DIR}/${TARGET}-${TESTS}" +DAILY_REPORTS_TESTS_DIR="${DAILY_REPORTS_DIR}/${TARGET}-${TESTS}" +DAILY_SCLORG_TESTS_DIR="${SHARED_DIR}/daily_scl_tests" + +DIR="${WORK_DIR}/${TARGET}-${TESTS}-${SET_TEST}" if [[ "$TESTS" == "test-upstream" ]]; then - DIR="${DAILY_TEST_DIR}/${TARGET}-${TESTS}" + DIR="${WORK_DIR}/${TARGET}-${TESTS}" fi -LOG_FILE="${LOGS_DIR}/${TARGET}-${TESTS}.log" +LOG_FILE="${LOCAL_LOGS_DIR}/${TARGET}-${TESTS}.log" + + +export USER_ID=$(id -u) +export GROUP_ID=$(id -g) + +function generate_passwd_file() { + grep -v ^ci-scripts /etc/passwd > "$HOME/passwd" + echo "ci-scripts:x:${USER_ID}:${GROUP_ID}:User for running ci-scripts:${HOME}:/bin/bash" >> "$HOME/passwd" + export LD_PRELOAD=libnss_wrapper.so + export NSS_WRAPPER_PASSWD=${HOME}/passwd + export NSS_WRAPPER_GROUP=/etc/group +} function move_logs_to_old() { echo "Moving logs to old directory" - if [[ -d "${LOGS_DIR_OLD}" ]]; then - rm -rf "${LOGS_DIR_OLD}/*" - fi - if [[ -d "${RESULTS_DIR_OLD}" ]]; then - rm -rf "${RESULTS_DIR_OLD}/*" - fi - mv "${LOG_FILE}/*" "${LOGS_DIR_OLD}/" - mv "${RESULTS_TARGET_DIR}/*" "${RESULTS_DIR_OLD}/" + mv "${DAILY_REPORTS_DIR}/*" "${DAILY_REPORTS_DIR}_old/" echo "Logs moved to old directory" } function prepare_environment() { - if [[ ! -d "${LOGS_DIR}" ]]; then - mkdir -p "${LOGS_DIR}" - fi - if [[ ! -d "${RESULTS_DIR}" ]]; then - mkdir -p "${RESULTS_DIR}" - fi - if [[ ! -d "${LOGS_DIR_OLD}" ]]; then - mkdir -p "${LOGS_DIR_OLD}" - fi - if [[ ! -d "${RESULTS_DIR_OLD}" ]]; then - mkdir -p "${RESULTS_DIR_OLD}" - fi - mkdir -p "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/results" - mkdir -p "$DIR" + mkdir -p "${LOCAL_LOGS_DIR}" + mkdir -p "${WORK_DIR}" + mkdir -p "${DIR}" + mkdir -p "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/results" + } function get_compose() { @@ -65,13 +65,15 @@ function get_compose() { elif [[ "$TARGET" == "fedora" ]]; then COMPOSE="1MT-Fedora-${VERSION}" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" - TFT_PLAN="nightly-container-f" + TFT_PLAN="nightly-container-fedora" elif [[ "$TARGET" == "c9s" ]]; then COMPOSE="1MT-CentOS-Stream-9" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" + TFT_PLAN="nightly-container-centos-stream-9" elif [[ "$TARGET" == "c10s" ]]; then COMPOSE="1MT-CentOS-Stream-10" TMT_PLAN_DIR="$UPSTREAM_TMT_DIR" + TFT_PLAN="nightly-container-centos-stream-10" else echo "This target is not supported" exit 1 @@ -85,36 +87,35 @@ function run_tests() { if [[ "$TESTS" != "test-upstream" ]]; then ENV_VARIABLES="$ENV_VARIABLES -e SET_TEST=$SET_TEST" fi - TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision --how minute --auto-select-network --image ${COMPOSE}" + TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision -v -v --how minute --auto-select-network --image ${COMPOSE}" echo "TMT command is: $TMT_COMMAND" | tee -a "${LOG_FILE}" touch "${RESULTS_TARGET_DIR}/tmt_running" set -o pipefail $TMT_COMMAND | tee -a "${LOG_FILE}" if [[ $? -ne 0 ]]; then echo "TMT command $TMT_COMMAND has failed." - if [[ -f "${RESULTS_TARGET_DIR}/tmt_success" ]]; then - rm -f "${RESULTS_TARGET_DIR}/tmt_success" + if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success" ]]; then + rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success" fi - touch "${RESULTS_TARGET_DIR}/tmt_failed" + touch "${DAILY_REPORTS_TESTS_DIR}/tmt_failed" else - if [[ -f "${RESULTS_TARGET_DIR}/tmt_failed" ]]; then + if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_failed" ]]; then echo "Previous test run has failed but this one has passed." else - touch "${RESULTS_TARGET_DIR}/tmt_success" + touch "${DAILY_REPORTS_TESTS_DIR}/tmt_success" fi fi + ls -laR "${DIR}/plans/${TFT_PLAN}/data/" > "$DAILY_SCLORG_TESTS_DIR/all_files_${TARGET}_${TESTS}.txt" + cp "${LOG_FILE}" "${DAILY_SCLORG_TESTS_DIR}/log_${TARGET}_${TESTS}.txt" if [[ -d "${DIR}/plans/${TFT_PLAN}/data" ]]; then - cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/" - cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/" + cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" + cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" fi - cp "${DIR}/log.txt" "${RESULTS_TARGET_DIR}/" + cp "${DIR}/log.txt" "${DAILY_REPORTS_TESTS_DIR}/" set +o pipefail - rm -f "${RESULTS_TARGET_DIR}/tmt_running" + rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_running" } - - - if [[ "$TESTS" != "test" ]] && [[ "$TESTS" != "test-pytest" ]] && [[ "$TESTS" != "test-upstream" ]] && [[ "$TESTS" != "test-openshift-pytest" ]] && [[ "$TESTS" != "test-openshift-4" ]]; then echo "This test scenario is not enabled." exit 1 @@ -123,10 +124,14 @@ fi CWD=$(pwd) cd "$HOME" || { echo "Could not switch to $HOME"; exit 1; } +generate_passwd_file +# chown -R "${USER_ID}":0 $HOME/ +# chown -R "${USER_ID}":0 $WORK_DIR/ + prepare_environment get_compose -move_logs_to_old +# move_logs_to_old date > "${LOG_FILE}" curl --insecure -L https://url.corp.redhat.com/fmf-data > "/tmp/fmf_data" @@ -140,4 +145,3 @@ echo "TARGET is: ${TARGET} and test is: ${TESTS}" | tee -a "${LOG_FILE}" run_tests cd "$CWD" || exit 1 -rm -rf "$WORK_DIR" diff --git a/show_logs.py b/show_logs.py new file mode 100644 index 0000000..b5f8677 --- /dev/null +++ b/show_logs.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import os +import sys + +from pathlib import Path + +DAILY_REPORTS_DIR = Path("/var/ci-scripts/daily_reports_dir/") +DAILY_SCL_TESTS_DIR = Path("/var/ci-scripts/daily_scl_tests/") + + +class PVCWatcherReport: + def __init__(self): + self.cwd = os.getcwd() + + def iter_results_in_directory(self): + """Yield all files in the given directory.""" + success_tests = [] + failed_tests = [] + for item in DAILY_REPORTS_DIR.iterdir(): + print(f"Inspecting item: {item}") + if item.is_dir(): + if (item / "tmt_success").exists(): + success_tests.append(item.name) + else: + failed_tests.append(item.name) + print(f"Success tests in {DAILY_REPORTS_DIR} are: {success_tests}") + print(f"Failed tests in {DAILY_REPORTS_DIR} are: {failed_tests}") + + def iter_over_executed_tests(self): + """Yield all executed tests in the given directory.""" + executed_tests = [] + for item in DAILY_SCL_TESTS_DIR.iterdir(): + print(f"Inspecting item: {item}") + if item.is_file(): + executed_tests.append(item.name) + print("Executed tests are:") + print(executed_tests.split("\n")) + + def print_report(self): + print("Daily SCL Tests Reports:") + self.iter_over_executed_tests() + self.iter_results_in_directory() + + +if __name__ == "__main__": + dgr = PVCWatcherReport() + dgr.print_report() + sys.exit(0) diff --git a/show_logs.sh b/show_logs.sh new file mode 100755 index 0000000..21234e7 --- /dev/null +++ b/show_logs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -x + +while true; do + echo "Displaying log files every 10 minutes..." + python3 /root/ci-scripts/show_logs.py + # Let's sleep for 10 minutes + sleep 600 +done