Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build-and-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ jobs:
registry_username: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_USERNAME }}
registry_token: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_TOKEN }}
dockerfile: Dockerfile.daily-tests
tag: "0.5.0"
tag: "0.6.0"
image_name: "upstream-daily-tests"
quay_application_token: ${{ secrets.QUAY_IMAGE_SCLORG_UPDATE_DESC }}
19 changes: 10 additions & 9 deletions Dockerfile.daily-tests
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
FROM quay.io/fedora/fedora:42

ENV WORK_DIR="/var/tmp" \
ENV SHARED_DIR="/var/ci-scripts" \
VERSION="42" \
RELEASE_UPSTREAM="0.5.0" \
DOWNSTREAM_TMT_DIR="sclorg-tmt-plans" \
RELEASE_UPSTREAM="0.6.0" \
UPSTREAM_TMT_REPO="https://github.com/sclorg/sclorg-testing-farm" \
UPSTREAM_TMT_DIR="sclorg-testing-farm" \
HOME="/home/nightly" \
SUMMARY="Daily tests for SCL org projects" \
DESCRIPTION="This image is used to run daily tests for SCL org projects in CI." \
NAME="nightly"
NAME="nightly" \
WORK_DIR="/home/nightly/ci-scripts"

LABEL summary="${SUMMARY}" \
description="${DESCRIPTION}" \
Expand All @@ -18,15 +18,16 @@ LABEL summary="${SUMMARY}" \
io.k8s.vendor="SCL org" \
architecture="x86_64"

COPY requirements.sh "${WORK_DIR}/requirements.sh"
RUN bash "${WORK_DIR}/requirements.sh"

RUN dnf install -y python3.13-pip git && \
RUN dnf install -y python3.13-pip git nss_wrapper && \
dnf clean all

RUN git clone "${UPSTREAM_TMT_REPO}" "${WORK_DIR}/${UPSTREAM_TMT_DIR}"
COPY requirements.sh "${WORK_DIR}/requirements.sh"
RUN bash "${WORK_DIR}/requirements.sh"
RUN cd "${WORK_DIR}" && git clone "${UPSTREAM_TMT_REPO}" "${UPSTREAM_TMT_DIR}"

COPY . /root/ci-scripts
WORKDIR "${HOME}"

# USER 900

CMD ["/root/ci-scripts/run_nightly_tests.sh"]
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ shellcheck:
./run-shellcheck.sh `git ls-files *.sh`

build_images:
podman build -t quay.io/sclorg/upstream-daily-tests:0.5.0 -f Dockerfile.daily-tests .
podman build -t quay.io/sclorg/upstream-daily-tests:0.6.0 -f Dockerfile.daily-tests .
4 changes: 2 additions & 2 deletions requirements.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@

set -ex

useradd -u 500 -r -g 0 -m -s /bin/bash -c "Default Application User" "${NAME}"
chown -R 500:0 "${HOME}"
useradd -u 900 -r -g 0 -m -s /bin/bash -c "Default Application User" "${NAME}"
chown -R 900:0 "${HOME}"
chmod -R a+rwx "${HOME}"
100 changes: 52 additions & 48 deletions run_nightly_tests.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

set -x

[[ -z "$TARGET" ]] && { echo "You have to specify target to build SCL images. rhel9, rhel8, or fedora" && exit 1 ; }
[[ -z "$TESTS" ]] && { echo "You have to specify type of the test to run. test, test-pytest, test-openshift, test-openshift-pytest" && exit 1 ; }
SET_TEST=""
Expand All @@ -8,48 +10,46 @@ if [[ "${TESTS}" != "test-upstream" ]]; then
SET_TEST="$TEST_TYPE"
fi

LOGS_DIR="${WORK_DIR}/daily_tests_logs/"
LOGS_DIR_OLD="${LOGS_DIR}/old"
DAILY_TEST_DIR="${WORK_DIR}/daily_scl_tests"
RESULTS_DIR="${WORK_DIR}/daily_reports_dir"
RESULTS_DIR_OLD="${RESULTS_DIR}/old"
SCRIPT="daily_scl_tests"
# Local working directories
WORK_DIR="${HOME}/ci-scripts/"
LOCAL_LOGS_DIR="${HOME}/logs/"

# Shared directories between runs
DAILY_REPORTS_DIR="${SHARED_DIR}/daily_reports_dir"
TFT_PLAN="nightly-container-$TARGET"
DIR="${DAILY_TEST_DIR}/${TARGET}-${TESTS}-${SET_TEST}"
RESULTS_TARGET_DIR="${RESULTS_DIR}/${TARGET}-${TESTS}"
DAILY_REPORTS_TESTS_DIR="${DAILY_REPORTS_DIR}/${TARGET}-${TESTS}"
DAILY_SCLORG_TESTS_DIR="${SHARED_DIR}/daily_scl_tests"

DIR="${WORK_DIR}/${TARGET}-${TESTS}-${SET_TEST}"
if [[ "$TESTS" == "test-upstream" ]]; then
DIR="${DAILY_TEST_DIR}/${TARGET}-${TESTS}"
DIR="${WORK_DIR}/${TARGET}-${TESTS}"
fi
LOG_FILE="${LOGS_DIR}/${TARGET}-${TESTS}.log"
LOG_FILE="${LOCAL_LOGS_DIR}/${TARGET}-${TESTS}.log"


export USER_ID=$(id -u)
export GROUP_ID=$(id -g)

function generate_passwd_file() {
grep -v ^ci-scripts /etc/passwd > "$HOME/passwd"
echo "ci-scripts:x:${USER_ID}:${GROUP_ID}:User for running ci-scripts:${HOME}:/bin/bash" >> "$HOME/passwd"
export LD_PRELOAD=libnss_wrapper.so
export NSS_WRAPPER_PASSWD=${HOME}/passwd
export NSS_WRAPPER_GROUP=/etc/group
}

function move_logs_to_old() {
echo "Moving logs to old directory"
if [[ -d "${LOGS_DIR_OLD}" ]]; then
rm -rf "${LOGS_DIR_OLD}/*"
fi
if [[ -d "${RESULTS_DIR_OLD}" ]]; then
rm -rf "${RESULTS_DIR_OLD}/*"
fi
mv "${LOG_FILE}/*" "${LOGS_DIR_OLD}/"
mv "${RESULTS_TARGET_DIR}/*" "${RESULTS_DIR_OLD}/"
mv "${DAILY_REPORTS_DIR}/*" "${DAILY_REPORTS_DIR}_old/"
echo "Logs moved to old directory"
}

function prepare_environment() {
if [[ ! -d "${LOGS_DIR}" ]]; then
mkdir -p "${LOGS_DIR}"
fi
if [[ ! -d "${RESULTS_DIR}" ]]; then
mkdir -p "${RESULTS_DIR}"
fi
if [[ ! -d "${LOGS_DIR_OLD}" ]]; then
mkdir -p "${LOGS_DIR_OLD}"
fi
if [[ ! -d "${RESULTS_DIR_OLD}" ]]; then
mkdir -p "${RESULTS_DIR_OLD}"
fi
mkdir -p "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/results"
mkdir -p "$DIR"
mkdir -p "${LOCAL_LOGS_DIR}"
mkdir -p "${WORK_DIR}"
mkdir -p "${DIR}"
mkdir -p "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/results"

}

function get_compose() {
Expand All @@ -65,13 +65,15 @@ function get_compose() {
elif [[ "$TARGET" == "fedora" ]]; then
COMPOSE="1MT-Fedora-${VERSION}"
TMT_PLAN_DIR="$UPSTREAM_TMT_DIR"
TFT_PLAN="nightly-container-f"
TFT_PLAN="nightly-container-fedora"
elif [[ "$TARGET" == "c9s" ]]; then
COMPOSE="1MT-CentOS-Stream-9"
TMT_PLAN_DIR="$UPSTREAM_TMT_DIR"
TFT_PLAN="nightly-container-centos-stream-9"
elif [[ "$TARGET" == "c10s" ]]; then
COMPOSE="1MT-CentOS-Stream-10"
TMT_PLAN_DIR="$UPSTREAM_TMT_DIR"
TFT_PLAN="nightly-container-centos-stream-10"
else
echo "This target is not supported"
exit 1
Expand All @@ -85,36 +87,35 @@ function run_tests() {
if [[ "$TESTS" != "test-upstream" ]]; then
ENV_VARIABLES="$ENV_VARIABLES -e SET_TEST=$SET_TEST"
fi
TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision --how minute --auto-select-network --image ${COMPOSE}"
TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision -v -v --how minute --auto-select-network --image ${COMPOSE}"
echo "TMT command is: $TMT_COMMAND" | tee -a "${LOG_FILE}"
touch "${RESULTS_TARGET_DIR}/tmt_running"
set -o pipefail
$TMT_COMMAND | tee -a "${LOG_FILE}"
if [[ $? -ne 0 ]]; then
echo "TMT command $TMT_COMMAND has failed."
if [[ -f "${RESULTS_TARGET_DIR}/tmt_success" ]]; then
rm -f "${RESULTS_TARGET_DIR}/tmt_success"
if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success" ]]; then
rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_success"
fi
touch "${RESULTS_TARGET_DIR}/tmt_failed"
touch "${DAILY_REPORTS_TESTS_DIR}/tmt_failed"
else
if [[ -f "${RESULTS_TARGET_DIR}/tmt_failed" ]]; then
if [[ -f "${DAILY_REPORTS_TESTS_DIR}/tmt_failed" ]]; then
echo "Previous test run has failed but this one has passed."
else
touch "${RESULTS_TARGET_DIR}/tmt_success"
touch "${DAILY_REPORTS_TESTS_DIR}/tmt_success"
fi
fi
ls -laR "${DIR}/plans/${TFT_PLAN}/data/" > "$DAILY_SCLORG_TESTS_DIR/all_files_${TARGET}_${TESTS}.txt"
cp "${LOG_FILE}" "${DAILY_SCLORG_TESTS_DIR}/log_${TARGET}_${TESTS}.txt"
if [[ -d "${DIR}/plans/${TFT_PLAN}/data" ]]; then
cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/"
cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${RESULTS_TARGET_DIR}/plans/${TFT_PLAN}/data/"
cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/"
cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/"
fi
cp "${DIR}/log.txt" "${RESULTS_TARGET_DIR}/"
cp "${DIR}/log.txt" "${DAILY_REPORTS_TESTS_DIR}/"
set +o pipefail
rm -f "${RESULTS_TARGET_DIR}/tmt_running"
rm -f "${DAILY_REPORTS_TESTS_DIR}/tmt_running"
}




if [[ "$TESTS" != "test" ]] && [[ "$TESTS" != "test-pytest" ]] && [[ "$TESTS" != "test-upstream" ]] && [[ "$TESTS" != "test-openshift-pytest" ]] && [[ "$TESTS" != "test-openshift-4" ]]; then
echo "This test scenario is not enabled."
exit 1
Expand All @@ -123,10 +124,14 @@ fi

CWD=$(pwd)
cd "$HOME" || { echo "Could not switch to $HOME"; exit 1; }
generate_passwd_file
# chown -R "${USER_ID}":0 $HOME/
# chown -R "${USER_ID}":0 $WORK_DIR/

prepare_environment
get_compose

move_logs_to_old
# move_logs_to_old

date > "${LOG_FILE}"
curl --insecure -L https://url.corp.redhat.com/fmf-data > "/tmp/fmf_data"
Expand All @@ -140,4 +145,3 @@ echo "TARGET is: ${TARGET} and test is: ${TESTS}" | tee -a "${LOG_FILE}"
run_tests

cd "$CWD" || exit 1
rm -rf "$WORK_DIR"
48 changes: 48 additions & 0 deletions show_logs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
import os
import sys

from pathlib import Path

DAILY_REPORTS_DIR = Path("/var/ci-scripts/daily_reports_dir/")
DAILY_SCL_TESTS_DIR = Path("/var/ci-scripts/daily_scl_tests/")


class PVCWatcherReport:
def __init__(self):
self.cwd = os.getcwd()

def iter_results_in_directory(self):
"""Yield all files in the given directory."""
success_tests = []
failed_tests = []
for item in DAILY_REPORTS_DIR.iterdir():
print(f"Inspecting item: {item}")
if item.is_dir():
if (item / "tmt_success").exists():
success_tests.append(item.name)
else:
failed_tests.append(item.name)
print(f"Success tests in {DAILY_REPORTS_DIR} are: {success_tests}")
print(f"Failed tests in {DAILY_REPORTS_DIR} are: {failed_tests}")

def iter_over_executed_tests(self):
"""Yield all executed tests in the given directory."""
executed_tests = []
for item in DAILY_SCL_TESTS_DIR.iterdir():
print(f"Inspecting item: {item}")
if item.is_file():
executed_tests.append(item.name)
print("Executed tests are:")
print(executed_tests.split("\n"))

def print_report(self):
print("Daily SCL Tests Reports:")
self.iter_over_executed_tests()
self.iter_results_in_directory()


if __name__ == "__main__":
dgr = PVCWatcherReport()
dgr.print_report()
sys.exit(0)
10 changes: 10 additions & 0 deletions show_logs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash

set -x

while true; do
echo "Displaying log files every 10 minutes..."
python3 /root/ci-scripts/show_logs.py
# Let's sleep for 10 minutes
sleep 600
done