runCI

#!/bin/bash

# This script will run the test suite. It requires no parameters, but needs
# some files to be present on the system. These are:
# - file containing the URL to report to
# - git repository with the code to compile & run

DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

# Enable coredump capture
ulimit -c unlimited
mkdir -p /tmp/coredumps
echo "/tmp/coredumps/core.%e.%p" | sudo tee /proc/sys/kernel/core_pattern > /dev/null

if [ ! -f "$DIR/variables" ]; then
    # No variable file defined
    sudo shutdown -h now
fi

# Functions for re-use in various stages of the test progress

# Post status to the server with retry logic
function postStatus {
    local status="$1"
    local message="$2"
    local max_retries=3
    local retry_delay=5
    local attempt=1

    echo "Posting ${status} - ${message} to the server:" >> "${logFile}"

    while [ $attempt -le $max_retries ]; do
        local http_code
        http_code=$(curl -s -A "${userAgent}" --data "type=progress&status=${status}&message=${message}" \
            -w "%{http_code}" -o /tmp/curl_response.txt "${reportURL}" 2>/dev/null)
        local curl_exit=$?

        if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
            echo "Status posted successfully (HTTP ${http_code})" >> "${logFile}"
            cat /tmp/curl_response.txt >> "${logFile}" 2>/dev/null
            echo "" >> "${logFile}"
            sleep 5
            return 0
        fi

        echo "Attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}"
        attempt=$((attempt + 1))

        if [ $attempt -le $max_retries ]; then
            echo "Retrying in ${retry_delay} seconds..." >> "${logFile}"
            sleep $retry_delay
            retry_delay=$((retry_delay * 2))
        fi
    done

    echo "ERROR: Failed to post status after ${max_retries} attempts" >> "${logFile}"
    return 1
}

# Send the log file to the server with retry logic
function sendLogFile {
    local max_retries=3
    local retry_delay=5
    local attempt=1

    echo "Sending log to the server:" >> "${logFile}"

    while [ $attempt -le $max_retries ]; do
        local http_code
        http_code=$(curl -s -A "${userAgent}" --form "type=logupload" --form "file=@${logFile}" \
            -w "%{http_code}" -o /tmp/curl_log_response.txt "${reportURL}" 2>/dev/null)
        local curl_exit=$?

        if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
            echo "Log uploaded successfully (HTTP ${http_code})" >> "${logFile}"
            sleep 5
            return 0
        fi

        echo "Log upload attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}"
        attempt=$((attempt + 1))

        if [ $attempt -le $max_retries ]; then
            echo "Retrying log upload in ${retry_delay} seconds..." >> "${logFile}"
            sleep $retry_delay
            retry_delay=$((retry_delay * 2))
        fi
    done

    echo "ERROR: Failed to upload log after ${max_retries} attempts" >> "${logFile}"
    return 1
}

# Exit script and post abort status
function haltAndCatchFire {
        sendLogFile
        postStatus "canceled" $1 >> "${logFile}"
        sudo shutdown -h now
}

# Fail when the exit status is not equal to 0
function executeCommand {
        #echo "$@"
        "$@" >> "${logFile}"
        local status=$?
        if [ ${status} -ne 0 ]; then
                haltAndCatchFire "" # No message needed as we post before anyway
        fi
}

# Source variables
. "$DIR/variables"

# Add cargo to path
PATH="/root/.cargo/bin:$PATH"

reportURL=$(curl http://metadata/computeMetadata/v1/instance/attributes/reportURL -H "Metadata-Flavor: Google")
userAgent="CCX/CI_BOT"
logFile="${reportFolder}/log.html"

postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact"

if [ -e "${dstDir}/ccextractor" ]; then
        cp $dstDir/* ./
        chmod 700 ccextractor
        chmod +x ${tester}
        # Log binary version for verification (commit SHA will be visible in output)
        echo "=== CCExtractor Binary Version ===" >> "${logFile}"
        ./ccextractor --version >> "${logFile}" 2>&1
        echo "=== End Version Info ===" >> "${logFile}"
        postStatus "testing" "Running tests"
        ccextractor_path="$(pwd)/ccextractor"
        combined_stdout="/tmp/combined_stdout.log"
        : > "${combined_stdout}"

        # Create a wrapper script that tees stdout/stderr to a combined log
        wrapper_path="$(pwd)/ccextractor_wrapper"
        cat > "${wrapper_path}" << 'WRAPPER_EOF'
#!/bin/bash
COMBINED_LOG="/tmp/combined_stdout.log"
REAL_BINARY="PLACEHOLDER_BINARY"
EXIT_CODE_FILE="/tmp/.wrapper_exit_code"
echo "=== TEST INVOCATION: $@ ===" >> "$COMBINED_LOG"
{ "$REAL_BINARY" "$@" 2>&1; echo $? > "$EXIT_CODE_FILE"; } | tee -a "$COMBINED_LOG"
exit_code=$(cat "$EXIT_CODE_FILE")
echo "=== EXIT CODE: ${exit_code} ===" >> "$COMBINED_LOG"
echo "" >> "$COMBINED_LOG"
exit $exit_code
WRAPPER_EOF
        sed -i "s|PLACEHOLDER_BINARY|${ccextractor_path}|" "${wrapper_path}"
        chmod +x "${wrapper_path}"

        executeCommand cd ${suiteDstDir}
        executeCommand ${tester} --debug --entries "${testFile}" --executable "${wrapper_path}" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}"

        # Upload AI artifacts to GCS
        gcs_bucket=$(curl -s "http://metadata/computeMetadata/v1/instance/attributes/bucket" -H "Metadata-Flavor: Google")
        test_id=$(curl -s "http://metadata/computeMetadata/v1/instance/attributes/testID" -H "Metadata-Flavor: Google")
        token=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" -H "Metadata-Flavor: Google" | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")

        upload_artifact() {
            local file_path="$1"
            local dest_path="$2"
            if [ -f "$file_path" ]; then
                local http_code
                http_code=$(curl -s -X POST --data-binary @"$file_path" \
                    -H "Authorization: Bearer $token" \
                    -H "Content-Type: application/octet-stream" \
                    -w "%{http_code}" \
                    -o /dev/null \
                    "https://storage.googleapis.com/upload/storage/v1/b/${gcs_bucket}/o?uploadType=media&name=${dest_path}")
                if [ -z "$http_code" ] || [ "$http_code" -ne 200 ]; then
                    echo "GCS upload failed for ${dest_path}: HTTP ${http_code:-no_response}" >> "${logFile}"
                fi
            fi
        }

        upload_artifact "$ccextractor_path" "test_artifacts/${test_id}/ccextractor"

        # Upload combined stdout log
        upload_artifact "${combined_stdout}" "test_artifacts/${test_id}/combined_stdout.log"

        # Upload coredumps if any
        for core_file in /tmp/coredumps/core.*; do
            if [ -f "$core_file" ]; then
                upload_artifact "$core_file" "test_artifacts/${test_id}/coredump"
                break
            fi
        done

        sendLogFile
        postStatus "completed" "Ran all tests"

        sudo shutdown -h now
else
        haltAndCatchFire "artifact"
fi