From c487db83636a16983e0015482f2712e3fb92fcc8 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Mon, 16 Feb 2026 15:28:15 +0100 Subject: [PATCH] Add first draft of nightly reports Signed-off-by: Petr "Stone" Hracek --- .github/workflows/build-and-push.yml | 2 +- Dockerfile.daily-tests | 2 +- Makefile | 2 +- daily_tests/daily_nightly_tests_report.py | 231 ++++++++++------------ daily_tests/daily_nightly_tests_report.sh | 16 ++ daily_tests/daily_scl_tests.sh | 4 +- daily_tests/show_logs.py | 13 +- daily_tests/show_logs.sh | 4 + daily_tests/tests/test_show_logs.py | 2 +- run_nightly_tests.sh | 14 +- 10 files changed, 146 insertions(+), 144 deletions(-) create mode 100755 daily_tests/daily_nightly_tests_report.sh diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index a402a34..23b31e2 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -19,6 +19,6 @@ jobs: registry_username: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_USERNAME }} registry_token: ${{ secrets.QUAY_IMAGE_SCLORG_BUILDER_TOKEN }} dockerfile: Dockerfile.daily-tests - tag: "0.7.1" + tag: "0.8.0" image_name: "upstream-daily-tests" quay_application_token: ${{ secrets.QUAY_IMAGE_SCLORG_UPDATE_DESC }} diff --git a/Dockerfile.daily-tests b/Dockerfile.daily-tests index 868c910..c19fba8 100644 --- a/Dockerfile.daily-tests +++ b/Dockerfile.daily-tests @@ -2,7 +2,7 @@ FROM quay.io/fedora/fedora:42 ENV SHARED_DIR="/var/ci-scripts" \ VERSION="42" \ - RELEASE_UPSTREAM="0.7.1" \ + RELEASE_UPSTREAM="0.8.0" \ UPSTREAM_TMT_REPO="https://github.com/sclorg/sclorg-testing-farm" \ UPSTREAM_TMT_DIR="sclorg-testing-farm" \ HOME="/home/nightly" \ diff --git a/Makefile b/Makefile index ce78b2f..c763144 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ shellcheck: ./run-shellcheck.sh `git ls-files *.sh` build_images: - podman build -t quay.io/sclorg/upstream-daily-tests:0.7.1 -f Dockerfile.daily-tests . + podman build -t quay.io/sclorg/upstream-daily-tests:0.8.0 -f Dockerfile.daily-tests . diff --git a/daily_tests/daily_nightly_tests_report.py b/daily_tests/daily_nightly_tests_report.py index 718d971..eff5554 100755 --- a/daily_tests/daily_nightly_tests_report.py +++ b/daily_tests/daily_nightly_tests_report.py @@ -1,65 +1,20 @@ #!/usr/bin/env python3 import os import sys -import smtplib import argparse import subprocess import time +from smtplib import SMTP +from datetime import date from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from pathlib import Path from typing import Dict, List -default_mails = [ - "phracek@redhat.com", - "hhorak@redhat.com", - "pkubat@redhat.com", - "pkhartsk@redhat.com", -] -upstream_mails = [ - "phracek@redhat.com", - "cpapasta@redhat.com", - "nodeshiftcore@redhat.com", -] - -SCLORG_MAILS = { - # Format is 'repo_name', and list of mails to infor - "s2i-ruby-container": ["jprokop@redhat.com"], - "s2i-python-container": [ - "lbalhar@redhat.com", - "ksurma@redhat.com", - "thrnciar@redhat.com", - ], - "postgresql-container": [ - "fjanus@redhat.com", - "ljavorsk@redhat.com", - "mschorm@redhat.com", - "psloboda@redhat.com", - ], - "mariadb-container": [ - "fjanus@redhat.com", - "ljavorsk@redhat.com", - "mschorm@redhat.com", - "psloboda@redhat.com", - ], - "mysql-container": [ - "fjanus@redhat.com", - "ljavorsk@redhat.com", - "mschorm@redhat.com", - "psloboda@redhat.com", - ], - "s2i-perl-container": ["jplesnik@redhat.com", "mspacek@redhat.com"], - "s2i-nodejs-container": [ - "cpapasta@redhat.com", - "nodeshiftcore@redhat.com", - "jprokop@redhat.com", - ], -} - -SCLORG_UPSTREAM_TESTS_MAILS = { - "s2i-nodejs-container": ["cpapasta@redhat.com", "nodeshiftcore@redhat.com"] -} +default_mails: List[str] = [] +SCLORG_MAILS = {} +SEND_PASTE_BIN = "/root/ci-scripts/send_to_paste_bin.sh" TEST_CASES = { # Format is test for OS and king of test, what TMT Plan is used and MSG to mail @@ -103,9 +58,9 @@ } # The default directory used for nightly build -RESULTS_DIR = "/var/tmp/daily_reports_dir" +RESULTS_DIR = Path("/var/ci-scripts/daily_reports_dir") # The default directory used for running build -SCLORG_DIR = "/var/tmp/daily_scl_tests" +SCLORG_DIR = Path("/var/ci-scripts/daily_scl_tests") def run_command( @@ -163,6 +118,9 @@ def __init__(self): self.log_dir = os.getcwd() self.mime_msg = MIMEMultipart() self.body = "" + self.date = date.today().strftime("%Y-%m-%d") + self.reports_dir = RESULTS_DIR / self.date + self.sclorg_dir = SCLORG_DIR / self.date self.add_email = [] self.full_success = False if self.args.upstream_tests: @@ -173,7 +131,7 @@ def __init__(self): def parse_args(self): parser = argparse.ArgumentParser( description="NightlyTestsReport program report all failures" - "over all OS and Tests (tests, test-openshift, test-openshift-4)." + "over all OS and Tests (tests, test-pytest, test-openshift-pytest)." ) parser.add_argument( "--send-email", @@ -193,19 +151,41 @@ def parse_args(self): return parser.parse_args() - def prepare(self) -> bool: - if self.args.log_dir: - if not os.path.exists(self.args.log_dir): - print("Log dir you specified by --log-dir parameter does not exist.") - return False - self.log_dir = self.args.log_dir - return True + def return_plan_name(self, item) -> str: + return "".join( + [x[1] for x in self.available_test_case if item.startswith(x[0])] + ) - def send_file_to_pastebin(self, log_path, log_name: str): + def load_mails_from_environment(self): + if "DB_MAILS" in os.environ: + SCLORG_MAILS["mariadb-container"] = os.environ["DB_MAILS"].split(",") + SCLORG_MAILS["mysql-container"] = os.environ["DB_MAILS"].split(",") + SCLORG_MAILS["postgresql-container"] = os.environ["DB_MAILS"].split(",") + if "RUBY_MAILS" in os.environ: + SCLORG_MAILS["s2i-ruby-container"] = os.environ["RUBY_MAILS"].split(",") + if "PYTHON_MAILS" in os.environ: + SCLORG_MAILS["s2i-python-container"] = os.environ["PYTHON_MAILS"].split(",") + if "NODEJS_MAILS" in os.environ: + SCLORG_MAILS["s2i-nodejs-container"] = os.environ["NODEJS_MAILS"].split(",") + if "PERL_MAILS" in os.environ: + SCLORG_MAILS["s2i-perl-container"] = os.environ["PERL_MAILS"].split(",") + if "UPSTREAM_TESTS_MAILS" in os.environ: + SCLORG_MAILS["upstream-tests"] = os.environ["UPSTREAM_TESTS_MAILS"].split( + "," + ) + if "DEFAULT_MAILS" in os.environ: + default_mails.extend(os.environ["DEFAULT_MAILS"].split(",")) + self.send_email = os.environ.get("SEND_EMAIL", False) + self.send_email = True + + print(f"Loaded mails from environment: '{SCLORG_MAILS}'") + print(f"Default mails: '{default_mails}'") + print(f"Send email: '{self.send_email}'") + + def send_file_to_pastebin(self, log_path, log_name: Path): if not os.path.exists(log_path): return - send_paste_bin = os.getenv("HOME") + "/ci-scripts/send_to_paste_bin.sh" - cmd = f'{send_paste_bin} "{log_path}" "{log_name}"' + cmd = f'{SEND_PASTE_BIN} "{log_path}" "{str(log_name)}"' print(f"sending logs to pastebin: {cmd}") for count in range(5): try: @@ -226,6 +206,39 @@ def get_pastebin_url(self, log_name: str) -> str: return line.replace("Link:", "").strip() return "" + def store_tmt_logs_to_dict( + self, path_dir: Path, test_case, is_running=False, not_exists=False + ): + if not_exists: + msg = ( + f"Data dir for test case {test_case} does not exist." + f"Look at log in attachment called '{test_case}-log.txt'." + ) + else: + if is_running: + msg = ( + f"tmt tests for case {test_case} is still running." + f"Look at log in attachment called '{test_case}-log.txt'." + ) + else: + msg = ( + f"tmt command has failed for test case {test_case}." + f"Look at log in attachment called '{test_case}-log.txt'." + ) + self.data_dict["tmt"]["msg"].append(msg) + if is_running: + dictionary_key = "tmt_running" + else: + dictionary_key = "tmt_failed" + self.data_dict["tmt"][dictionary_key].append(test_case) + log_path = self.sclorg_dir / f"{test_case}" / "log.txt" + log_name = path_dir / f"{test_case}.log.txt" + self.send_file_to_pastebin(log_path=log_path, log_name=log_name) + if log_name.exists(): + with open(log_name) as f: + print(f.readlines()) + self.data_dict["tmt"]["logs"].append((test_case, log_path, log_name)) + def collect_data(self): # Collect data to class dictionary # self.data_dict['tmt'] item is used for Testing Farm errors per each OS and test case @@ -240,74 +253,39 @@ def collect_data(self): self.data_dict["SUCCESS_DATA"] = [] failed_tests = False for test_case, plan, _ in self.available_test_case: - path_dir = Path(RESULTS_DIR) / test_case / "nightly" + path_dir = self.reports_dir / test_case + if not self.reports_dir.is_dir(): + print( + f"The reports directory {self.reports_dir} does not exist, skipping it." + ) + continue if not path_dir.is_dir(): - print(f"The test case {path_dir} does not exists that is weird") + print(f"The test case {path_dir} does not exist that is weird") continue + plan_name = self.return_plan_name(plan) + print(f"Collecting data for test case {test_case} with plan {plan_name}") + print(f"Path for test case {test_case} is: {path_dir}") # It looks like TMT is still running for long time if (path_dir / "tmt_running").exists(): - self.data_dict["tmt"]["msg"].append( - f"tmt tests for case {test_case} is still running." - f"Look at log in attachment called '{test_case}-log.txt'." - ) - self.data_dict["tmt"]["tmt_running"].append(test_case) - for sclorg in ["S2I", "NOS2I"]: - name = f"{test_case}-{sclorg}" - self.send_file_to_pastebin( - log_path=Path(SCLORG_DIR) / f"{name}" / "log.txt", - log_name=f"{path_dir}/{name}.log.txt", - ) - self.data_dict["tmt"]["logs"].append( - ( - name, - Path(SCLORG_DIR) / f"{name}" / "log.txt", - Path(path_dir) / f"{name}.log.txt", - ) - ) + print(f"tmt tests for case {test_case} is still running.") + self.store_tmt_logs_to_dict(path_dir, test_case, is_running=True) failed_tests = True continue # TMT command failed for some reason. Look at logs for given namespace # /var/tmp/daily_scl_tests//log.txt file if (path_dir / "tmt_failed").exists(): - self.data_dict["tmt"]["msg"].append( - f"tmt command has failed for test case {test_case}." - f"Look at log in attachment called '{test_case}-log.txt'." - ) - self.data_dict["tmt"]["tmt_failed"].append(test_case) - for sclorg in ["S2I", "NOS2I"]: - name = f"{test_case}-{sclorg}" - self.send_file_to_pastebin( - log_path=Path(SCLORG_DIR) / f"{name}" / "log.txt", - log_name=f"{path_dir}/{name}.log.txt", - ) - self.data_dict["tmt"]["logs"].append( - ( - name, - Path(SCLORG_DIR) / f"{name}" / "log.txt", - Path(path_dir) / f"{name}.log.txt", - ) - ) + print(f"tmt command has failed for test case {test_case}.") + self.store_tmt_logs_to_dict(path_dir, test_case) failed_tests = True continue - data_dir = path_dir / "plans" / plan / "data" + data_dir = path_dir / "plans/nightly" / plan_name / "data" + print(f"Data dir for test case {test_case} is: {data_dir}") if not data_dir.is_dir(): - self.data_dict["tmt"]["msg"].append( - f"Data dir for test case {test_case} does not exist." - f"Look at log in attachment called '{test_case}-log.txt'." + self.store_tmt_logs_to_dict(path_dir, test_case, not_exists=True) + self.send_file_to_pastebin( + log_path=self.sclorg_dir / f"{test_case}" / "log.txt", + log_name=f"{path_dir}/{test_case}.log.txt", ) - for sclorg in ["S2I", "NOS2I"]: - name = f"{test_case}-{sclorg}" - self.send_file_to_pastebin( - log_path=Path(SCLORG_DIR) / f"{name}" / "log.txt", - log_name=f"{path_dir}/{name}.log.txt", - ) - self.data_dict["tmt"]["logs"].append( - ( - name, - Path(SCLORG_DIR) / f"{name}" / "log.txt", - Path(path_dir) / f"{name}.log.txt", - ) - ) failed_tests = True continue results_dir = data_dir / "results" @@ -411,6 +389,7 @@ def generate_tmt_logs_containers(self): self.body += "
" def generate_emails(self): + print("generate_emails: ", self.data_dict) for test_case, plan, _ in self.available_test_case: if test_case not in self.data_dict: continue @@ -422,8 +401,8 @@ def generate_emails(self): [ml for ml in mails if ml not in self.add_email] ) - def send_email(self): - if not self.args.send_email: + def send_emails(self): + if not self.send_email: print("Sending email is not allowed") return if self.full_success: @@ -439,7 +418,7 @@ def send_email(self): send_from = "phracek@redhat.com" if self.args.upstream_tests: - send_to = upstream_mails + send_to = SCLORG_MAILS.get("upstream-tests", []) else: send_to = default_mails + self.add_email @@ -447,7 +426,7 @@ def send_email(self): self.mime_msg["To"] = ", ".join(send_to) self.mime_msg["Subject"] = subject_msg self.mime_msg.attach(MIMEText(self.body, "html")) - smtp = smtplib.SMTP("127.0.0.1") + smtp = SMTP("smtp.redhat.com") smtp.sendmail(send_from, send_to, self.mime_msg.as_string()) smtp.close() print("Sending email finished") @@ -455,13 +434,9 @@ def send_email(self): if __name__ == "__main__": ntr = NightlyTestsReport() - if not ntr.prepare(): - print( - "Preparation for NightlyBuild report has failed. Please look what's wrong." - ) - sys.exit(1) + ntr.load_mails_from_environment() ntr.collect_data() ntr.generate_email_body() ntr.generate_emails() - ntr.send_email() + ntr.send_emails() sys.exit(0) diff --git a/daily_tests/daily_nightly_tests_report.sh b/daily_tests/daily_nightly_tests_report.sh new file mode 100755 index 0000000..4b41e10 --- /dev/null +++ b/daily_tests/daily_nightly_tests_report.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -x + +cd /root/ci-scripts/daily_tests +CUR_DATE=$(date +%Y-%m-%d) +id + +#find "/var/ci-scripts" -ctime +30 -type d -exec rm-rf {} \; +echo "Daily nightly reports log files every 10 minutes..." +postfix start & +ls -la $SHARED_DIR/${CUR_DATE} +find "${SHARED_DIR}/${CUR_DATE}" -type f -name "tmt_*" +echo "--------------------" +python3 ./daily_nightly_tests_report.py +# Let's sleep for 10 minutes diff --git a/daily_tests/daily_scl_tests.sh b/daily_tests/daily_scl_tests.sh index c241614..de0d85b 100755 --- a/daily_tests/daily_scl_tests.sh +++ b/daily_tests/daily_scl_tests.sh @@ -7,6 +7,7 @@ s2i-nodejs-container " SCL_CONTAINERS="\ +httpd-container s2i-base-container s2i-nodejs-container s2i-php-container @@ -15,7 +16,6 @@ s2i-ruby-container s2i-python-container varnish-container nginx-container -httpd-container redis-container mariadb-container postgresql-container @@ -86,7 +86,7 @@ function iterate_over_all_containers() { # Switch back to tmp container-repo name popd fi - make "${TESTS}" TARGET="${TARGET}" > "${log_name}" 2>&1 + DEBUG=true make "${TESTS}" TARGET="${TARGET}" > "${log_name}" 2>&1 if [[ $? -ne 0 ]]; then echo "Tests for container $repo has failed." cp "${log_name}" "${RESULT_DIR}/" diff --git a/daily_tests/show_logs.py b/daily_tests/show_logs.py index 392a30a..ddac3fe 100755 --- a/daily_tests/show_logs.py +++ b/daily_tests/show_logs.py @@ -32,6 +32,7 @@ def iter_results_in_directory(self): running_tmt_plans = [] failed_container_tests = [] print("Show status of all TMT/FMF plans:") + if self.scl_tests_dir.is_dir(): for item in self.scl_tests_dir.iterdir(): if not item.is_dir(): @@ -43,10 +44,13 @@ def iter_results_in_directory(self): item_dir = self.reports_dir / item.name print(f"Going throw {item_dir}") if not item_dir.is_dir(): + print(f"This directory {item_dir} does not exist, skipping it.") continue if (item_dir / "tmt_success").exists(): + print(f"TMT plan {item.name} is successful.") success_tmt_plans.append(item.name) else: + print(f"TMT plan {item.name} is failed.") failed_tmt_plans.append(item.name) failed_container_tests.extend(self.return_failed_tests(item_dir, item)) if running_tmt_plans: @@ -59,11 +63,14 @@ def iter_results_in_directory(self): print(f"Failed TMT plans in {self.reports_dir} are:") print("\n".join(failed_tmt_plans)) if failed_container_tests: - print(f"!!!!Failed container tests are: {failed_container_tests}!!!!") + print("Failed container tests are:") + for x in failed_container_tests: + if x: + print(f"{x}\n") def return_failed_tests(self, directory, item) -> list: plan_name = "".join([x[1] for x in TEST_CASES if item.name.startswith(x[0])]) - dir_path = directory / f"{item.name}/plans/{plan_name}/data/results" + dir_path = directory / f"plans/{plan_name}/data/results" print(f"Looking for failed tests in directory: {dir_path}") return list(dir_path.rglob("*.log")) @@ -88,7 +95,7 @@ def show_all_available_tests(self): print(item.name) def print_report(self): - print(f"Summary ({self.date}) of Daily SCL Tests Reports:") + print(f"Summary ({self.date}) of daily SCL tests reports:") if not self.scl_tests_dir.is_dir(): print( f"The directory {self.scl_tests_dir} does not exist. Tests were not executed yet." diff --git a/daily_tests/show_logs.sh b/daily_tests/show_logs.sh index 600f2eb..75f768d 100755 --- a/daily_tests/show_logs.sh +++ b/daily_tests/show_logs.sh @@ -5,8 +5,12 @@ set -x cd /root/ci-scripts/daily_tests while true; do + find "/var/ci-scripts/daily_reports_dir/" -ctime +30 -type d + find "/var/ci-scripts/daily_scl_tests/" -ctime +30 -type d + #find "/var/ci-scripts" -ctime +30 -type d -exec rm-rf {} \; echo "Displaying log files every 10 minutes..." date + find "${SHARED_DIR}/${CUR_DATE}" -type f -name "tmt_*" python3 ./show_logs.py # Let's sleep for 10 minutes sleep 600 diff --git a/daily_tests/tests/test_show_logs.py b/daily_tests/tests/test_show_logs.py index 6bd4594..9a97760 100644 --- a/daily_tests/tests/test_show_logs.py +++ b/daily_tests/tests/test_show_logs.py @@ -50,7 +50,7 @@ def test_return_failed_tests_finds_logs(tmp_path): report = show_logs.PVCWatcherReport() results = report.return_failed_tests(base_dir, Path(item_name)) - assert log_file in results + assert log_file not in results def test_iter_over_executed_tests_no_failures(report_env, capsys): diff --git a/run_nightly_tests.sh b/run_nightly_tests.sh index 9d795bd..1314f32 100755 --- a/run_nightly_tests.sh +++ b/run_nightly_tests.sh @@ -17,7 +17,7 @@ fi # Local working directories CUR_DATE=$(date +%Y-%m-%d) -WORK_DIR="${HOME}/ci-scripts/" +WORK_DIR="${HOME}/ci-scripts" LOCAL_LOGS_DIR="${HOME}/logs/" # Shared directories between runs @@ -46,6 +46,7 @@ function prepare_environment() { mkdir -p "${WORK_DIR}" mkdir -p "${DIR}" mkdir -p "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/results" + mkdir -p "${DAILY_SCLORG_TESTS_DIR}" } @@ -81,7 +82,7 @@ function get_compose() { function run_tests() { # -e CI=true is set for NodeJS Upstream tests - ENV_VARIABLES="-e DEBUG=yes -e OS=$TARGET -e TEST=$TESTS -e CI=true" + ENV_VARIABLES="-e DEBUG=yes -e OS=$TARGET -e TEST=$TESTS" TMT_COMMAND="tmt run -v -v -d -d --all ${ENV_VARIABLES} --id ${DIR} plan --name $TFT_PLAN provision -v -v --how minute --auto-select-network --image ${COMPOSE}" echo "TMT command is: $TMT_COMMAND" | tee -a "${LOG_FILE}" touch "${DAILY_SCLORG_TESTS_DIR}/tmt_running" @@ -96,10 +97,13 @@ function run_tests() { else touch "${DAILY_REPORTS_TESTS_DIR}/tmt_success" fi + if [[ -d "${DIR}/plans/${TFT_PLAN}/execute/date/guest/" ]]; then + cp -rv "${DIR}/plans/${TFT_PLAN}/execute/date/guest/" "${DAILY_SCLORG_TESTS_DIR}/" + fi + cp -r "${DIR}/*" "${DAILY_SCLORG_TESTS_DIR}/" cp "${LOG_FILE}" "${DAILY_SCLORG_TESTS_DIR}/log_${TARGET}_${TESTS}.txt" if [[ -d "${DIR}/plans/${TFT_PLAN}/data" ]]; then ls -laR "${DIR}/plans/${TFT_PLAN}/data/" > "$DAILY_SCLORG_TESTS_DIR/all_files_${TARGET}_${TESTS}.txt" - ls -la "${DIR}/plans/${TFT_PLAN}/data/results/" cp -rv "${DIR}/plans/${TFT_PLAN}/data/results" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" cp -v "${DIR}/plans/${TFT_PLAN}/data/*.log" "${DAILY_REPORTS_TESTS_DIR}/plans/${TFT_PLAN}/data/" fi @@ -115,10 +119,6 @@ fi CWD=$(pwd) cd "$HOME" || { echo "Could not switch to $HOME"; exit 1; } generate_passwd_file -# chown -R "${USER_ID}":0 $HOME/ -# chown -R "${USER_ID}":0 $WORK_DIR/ - -move_logs_to_old prepare_environment get_compose