run_coverage_ci.sh

#!/bin/bash
# Script to generate coverage for CI pipeline with timeouts to prevent hanging

set -e  # Exit on error
set -x  # Print commands for debugging

echo "Starting coverage generation for CI..."

# Set up coverage directory
mkdir -p coverage_html

# Set environment variables for CI 
export CI_EXIT_ON_TEST_FAILURE=1  # Exit on test failures to ensure code quality
export CI_TEST_TIMEOUT=60  # Default timeout

# Special handling for GitHub Actions environment
if [ -n "$GITHUB_WORKSPACE" ]; then
  echo "Running in GitHub Actions environment"
  echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE"
  echo "Current directory: $(pwd)"
  echo "Directory contents:"
  ls -la
fi

# Determine test directory
TEST_DIR=${TEST_DIR_ENV:-"test_dir"}
echo "Using test directory: $TEST_DIR"

# Try different locations if test directory not found
if [ ! -d "$TEST_DIR" ]; then
    echo "Warning: Test directory $TEST_DIR not found in current directory"
    
    # Try parent directory
    if [ -d "../$TEST_DIR" ]; then
        TEST_DIR="../$TEST_DIR"
        echo "Found test directory in parent directory: $TEST_DIR"
    # Try in GitHub workspace
    elif [ -n "$GITHUB_WORKSPACE" ] && [ -d "$GITHUB_WORKSPACE/$TEST_DIR" ]; then
        TEST_DIR="$GITHUB_WORKSPACE/$TEST_DIR"
        echo "Found test directory in GITHUB_WORKSPACE: $TEST_DIR"
    # Use find to locate test directory
    else
        TEST_DIR_FOUND=$(find . -type d -name "test_dir" | head -1)
        if [ -n "$TEST_DIR_FOUND" ]; then
            TEST_DIR="$TEST_DIR_FOUND"
            echo "Found test directory using find: $TEST_DIR"
        else
            echo "Error: Could not find test directory"
            echo "Current directory: $(pwd)"
            echo "Available directories:"
            ls -la
            # Continue anyway to avoid failing the CI
            # We'll handle individual files not found later
        fi
    fi
fi

# Set timeout duration (in seconds) from environment variable or use default
CI_TIMEOUT=${CI_TEST_TIMEOUT:-60}
echo "Using test timeout of $CI_TIMEOUT seconds (set CI_TEST_TIMEOUT env var to change)"

# Set up logging
LOG_DIR="test_logs"
mkdir -p "$LOG_DIR"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
SUMMARY_LOG="$LOG_DIR/test_summary_${TIMESTAMP}.log"
echo "Test run started at $(date)" > "$SUMMARY_LOG"
echo "Timeout value: $CI_TIMEOUT seconds" >> "$SUMMARY_LOG"
echo "Test directory: $TEST_DIR" >> "$SUMMARY_LOG"

# Function to handle test errors with better reporting
handle_test_error() {
  TEST_FILE=$1
  EXIT_CODE=$2
  LOG_FILE=$3
  
  echo "----------------------------------------" | tee -a "$SUMMARY_LOG"
  if [ $EXIT_CODE -eq 124 ]; then
    echo "⚠️ WARNING: $TEST_FILE TIMED OUT (after $CI_TIMEOUT seconds)" | tee -a "$SUMMARY_LOG"
  else
    echo "⚠️ WARNING: $TEST_FILE FAILED with exit code $EXIT_CODE" | tee -a "$SUMMARY_LOG"
  fi
  
  # If we have a log file, show the last few lines
  if [ -f "$LOG_FILE" ]; then
    echo "Last 10 lines from log:" | tee -a "$SUMMARY_LOG"
    tail -10 "$LOG_FILE" | tee -a "$SUMMARY_LOG"
  fi
  echo "----------------------------------------" | tee -a "$SUMMARY_LOG"
}

# Clean up any pycache files to avoid import conflicts
find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
find . -name "*.pyc" -delete

# Run tests in smaller batches with timeouts
echo "Running test suite with coverage enabled..."

# Define the basic tools tests paths
TOOLS_TESTS=(
  "$TEST_DIR/test_file_tools.py"
  "$TEST_DIR/test_system_tools.py"
  "$TEST_DIR/test_directory_tools.py"
  "$TEST_DIR/improved/test_quality_tools.py"
  "$TEST_DIR/improved/test_summarizer_tool.py"
  "$TEST_DIR/improved/test_tree_tool.py"
  "tests/tools/test_base_tool.py"
)

# Check if tools test files exist
TOOLS_TESTS_EXISTING=()
for TEST_FILE in "${TOOLS_TESTS[@]}"; do
  if [ -f "$TEST_FILE" ]; then
    TOOLS_TESTS_EXISTING+=("$TEST_FILE")
  else
    echo "Warning: Test file $TEST_FILE not found" | tee -a "$SUMMARY_LOG"
  fi
done

# First, run the basic tools tests which are known to work
if [ ${#TOOLS_TESTS_EXISTING[@]} -gt 0 ]; then
  echo "Running tools tests (known to work well)..." | tee -a "$SUMMARY_LOG"
  python -m pytest \
    --cov=src.cli_code \
    --cov-report=xml:coverage.xml \
    --cov-report=html:coverage_html \
    --cov-report=term \
    --timeout=$CI_TIMEOUT \
    "${TOOLS_TESTS_EXISTING[@]}"
else
  echo "No tools tests found to run" | tee -a "$SUMMARY_LOG"
  # Initialize coverage file to avoid errors
  python -m pytest \
    --cov=src.cli_code \
    --cov-report=xml:coverage.xml \
    --cov-report=html:coverage_html \
    --cov-report=term
fi

# Define model tests paths
MODEL_TESTS=(
  "$TEST_DIR/test_models_base.py"
  "$TEST_DIR/test_model_basic.py"
  "$TEST_DIR/test_model_integration.py"
)

# Check if model test files exist
MODEL_TESTS_EXISTING=()
for TEST_FILE in "${MODEL_TESTS[@]}"; do
  if [ -f "$TEST_FILE" ]; then
    MODEL_TESTS_EXISTING+=("$TEST_FILE")
  else
    echo "Warning: Test file $TEST_FILE not found" | tee -a "$SUMMARY_LOG"
  fi
done

# Now run the model tests separately
if [ ${#MODEL_TESTS_EXISTING[@]} -gt 0 ]; then
  echo "Running model tests..." | tee -a "$SUMMARY_LOG"
  python -m pytest \
    --cov=src.cli_code \
    --cov-append \
    --cov-report=xml:coverage.xml \
    --cov-report=html:coverage_html \
    --cov-report=term \
    --timeout=$CI_TIMEOUT \
    "${MODEL_TESTS_EXISTING[@]}"
else
  echo "No model tests found to run" | tee -a "$SUMMARY_LOG"
fi

# Track failures
FAILED_TESTS=0
TIMED_OUT_TESTS=0

# Function to run tests with a common pattern
run_test_group() {
  GROUP_NAME=$1
  shift
  TEST_FILES=("$@")
  
  echo "Running $GROUP_NAME tests..." | tee -a "$SUMMARY_LOG"
  
  for test_file in "${TEST_FILES[@]}"; do
    # Check if file exists
    if [ ! -f "$test_file" ]; then
      echo "Warning: Test file $test_file not found, skipping" | tee -a "$SUMMARY_LOG"
      continue
    fi
    
    echo "Running $test_file with timeout $CI_TIMEOUT seconds..." | tee -a "$SUMMARY_LOG"
    LOG_FILE="$LOG_DIR/$(basename $test_file).log"
    
    # Run test with timeout and capture output
    python -m pytest \
      --cov=src.cli_code \
      --cov-append \
      --timeout=$CI_TIMEOUT \
      "$test_file" > "$LOG_FILE" 2>&1
    
    EXIT_CODE=$?
    if [ $EXIT_CODE -ne 0 ]; then
      if [ $EXIT_CODE -eq 124 ]; then
        TIMED_OUT_TESTS=$((TIMED_OUT_TESTS + 1))
      else
        FAILED_TESTS=$((FAILED_TESTS + 1))
      fi
      handle_test_error "$test_file" "$EXIT_CODE" "$LOG_FILE"
    else
      echo "✅ $test_file completed successfully" | tee -a "$SUMMARY_LOG"
    fi
  done
}

# Run gemini model tests individually
run_test_group "gemini model" \
  "$TEST_DIR/test_gemini_model.py" \
  "$TEST_DIR/test_gemini_model_advanced.py" \
  "$TEST_DIR/test_gemini_model_coverage.py" \
  "$TEST_DIR/test_gemini_model_error_handling.py"

# Run ollama model tests individually
run_test_group "ollama model" \
  "$TEST_DIR/test_ollama_model.py" \
  "$TEST_DIR/test_ollama_model_advanced.py" \
  "$TEST_DIR/test_ollama_model_coverage.py" \
  "$TEST_DIR/test_ollama_model_context.py" \
  "$TEST_DIR/test_ollama_model_error_handling.py"

# Run config tests individually
run_test_group "config" \
  "$TEST_DIR/test_config.py" \
  "$TEST_DIR/test_config_comprehensive.py" \
  "$TEST_DIR/test_config_edge_cases.py" \
  "$TEST_DIR/test_config_missing_methods.py"

# Run main tests individually
run_test_group "main" \
  "$TEST_DIR/test_main.py" \
  "$TEST_DIR/test_main_comprehensive.py" \
  "$TEST_DIR/test_main_edge_cases.py" \
  "$TEST_DIR/test_main_improved.py"

# Run remaining tests individually
run_test_group "remaining" \
  "$TEST_DIR/test_task_complete_tool.py" \
  "$TEST_DIR/test_tools_base.py" \
  "$TEST_DIR/test_tools_init_coverage.py" \
  "$TEST_DIR/test_utils.py" \
  "$TEST_DIR/test_utils_comprehensive.py" \
  "$TEST_DIR/test_test_runner_tool.py" \
  "$TEST_DIR/test_basic_functions.py" \
  "$TEST_DIR/test_tools_basic.py" \
  "$TEST_DIR/test_tree_tool_edge_cases.py"

# Generate a final coverage report
echo "Generating final coverage report..." | tee -a "$SUMMARY_LOG"
python -m pytest \
  --cov=src.cli_code \
  --cov-report=xml:coverage.xml \
  --cov-report=html:coverage_html \
  --cov-report=term

echo "Coverage report generated in coverage.xml and coverage_html/" | tee -a "$SUMMARY_LOG"

# Print summary of test results
echo "" | tee -a "$SUMMARY_LOG"
echo "Test Summary:" | tee -a "$SUMMARY_LOG"
echo "-------------" | tee -a "$SUMMARY_LOG"
echo "Failed tests: $FAILED_TESTS" | tee -a "$SUMMARY_LOG"
echo "Timed out tests: $TIMED_OUT_TESTS" | tee -a "$SUMMARY_LOG"
echo "Log files available in: $LOG_DIR" | tee -a "$SUMMARY_LOG"
echo "Summary log: $SUMMARY_LOG" | tee -a "$SUMMARY_LOG"

# Extract overall coverage percentage for GitHub output
if [ -f "coverage.xml" ]; then
  echo "✅ coverage.xml file exists" | tee -a "$SUMMARY_LOG"
  
  # Extract overall coverage percentage
  COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); line_rate = float(root.attrib['line-rate'])*100; print('{:.2f}%'.format(line_rate))")
  echo "Overall coverage percentage: $COVERAGE" | tee -a "$SUMMARY_LOG"
  
  # Set output for GitHub Actions
  if [ -n "$GITHUB_OUTPUT" ]; then
    echo "percentage=$COVERAGE" >> $GITHUB_OUTPUT
  else
    echo "Note: GITHUB_OUTPUT not defined, skipping GitHub output" | tee -a "$SUMMARY_LOG"
  fi
else
  echo "❌ coverage.xml file not generated!" | tee -a "$SUMMARY_LOG"
  if [ -n "$GITHUB_OUTPUT" ]; then
    echo "percentage=0.00%" >> $GITHUB_OUTPUT
  fi
fi

echo "Coverage generation for CI completed." | tee -a "$SUMMARY_LOG"

# Determine exit code based on errors and CI environment
# In CI we might want to exit gracefully for some failures
CI_EXIT_ON_TEST_FAILURE=${CI_EXIT_ON_TEST_FAILURE:-1}

if [ $FAILED_TESTS -gt 0 -o $TIMED_OUT_TESTS -gt 0 ]; then
  echo "Test run had $FAILED_TESTS failing tests and $TIMED_OUT_TESTS timed out tests" | tee -a "$SUMMARY_LOG"
  
  if [ -n "$CI" ] && [ "$CI_EXIT_ON_TEST_FAILURE" = "1" ]; then
    echo "Exiting with error code due to test failures" | tee -a "$SUMMARY_LOG"
    exit 1
  else
    echo "Warning: Tests failed but continuing (CI_EXIT_ON_TEST_FAILURE=$CI_EXIT_ON_TEST_FAILURE)" | tee -a "$SUMMARY_LOG"
  fi
fi

# If we made it here, exit successfully
exit 0