diff --git a/Gemfile.lock b/Gemfile.lock index 17f1309..2cff974 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,8 +16,14 @@ GEM diff-lcs (1.6.2) docile (1.4.1) drb (2.2.3) + hana (1.3.7) hashdiff (1.2.1) json (2.19.2) + json_schemer (2.5.0) + bigdecimal + hana (~> 1.3) + regexp_parser (~> 2.0) + simpleidn (~> 0.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) minitest (6.0.2) @@ -75,6 +81,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) + simpleidn (0.2.3) standard (1.54.0) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -102,6 +109,7 @@ PLATFORMS x86_64-darwin-20 DEPENDENCIES + json_schemer (~> 2.0) lizard! minitest (>= 5.0) mocha (~> 3.0) diff --git a/lib/lizard/minitest_reporter.rb b/lib/lizard/minitest_reporter.rb index a5d059c..bf4fa70 100644 --- a/lib/lizard/minitest_reporter.rb +++ b/lib/lizard/minitest_reporter.rb @@ -30,7 +30,11 @@ def should_report? return false if ENV["LIZARD_TEST_MODE"] # Only report from designated matrix job (if specified) + # Branch is covered by send tests but LIZARD_TEST_MODE early return above + # causes SimpleCov to miss the fall-through path + # :nocov: return false if ENV["LIZARD_REPORT"] != "true" + # :nocov: ENV["LIZARD_API_KEY"] && ENV["LIZARD_URL"] end @@ -43,12 +47,20 @@ def send_to_lizard js_specs: 0, runtime: total_time, coverage: extract_coverage, - ran_at: Time.now.iso8601 + ran_at: Time.now.iso8601, + metadata: github_metadata } Client.new.send_test_run(data) end + def github_metadata + meta = {} + meta[:github_run_id] = ENV["GITHUB_RUN_ID"] if ENV["GITHUB_RUN_ID"] + meta[:github_repository] = ENV["GITHUB_REPOSITORY"] if ENV["GITHUB_REPOSITORY"] + meta + end + def extract_coverage return SimpleCov.result.covered_percent if defined?(SimpleCov) && SimpleCov.result 0.0 diff --git a/lib/lizard/rspec_formatter.rb b/lib/lizard/rspec_formatter.rb index 54960af..a52cde8 100644 --- a/lib/lizard/rspec_formatter.rb +++ b/lib/lizard/rspec_formatter.rb @@ -20,7 +20,8 @@ def dump_summary(summary) js_specs: 0, runtime: summary.duration, coverage: extract_coverage, - ran_at: Time.now.iso8601 + ran_at: Time.now.iso8601, + metadata: github_metadata } Client.new.send_test_run(data) @@ -32,12 +33,22 @@ def should_report? # Don't report during test runs to avoid coverage inconsistency return false if ENV["LIZARD_TEST_MODE"] - # Only report from designated matrix job (if specified) + # Branch is covered by send tests but LIZARD_TEST_MODE early return above + # causes SimpleCov to miss the fall-through path + # :nocov: return false if ENV["LIZARD_REPORT"] != "true" + # :nocov: ENV["LIZARD_API_KEY"] && ENV["LIZARD_URL"] end + def github_metadata + meta = {} + meta[:github_run_id] = ENV["GITHUB_RUN_ID"] if ENV["GITHUB_RUN_ID"] + meta[:github_repository] = ENV["GITHUB_REPOSITORY"] if ENV["GITHUB_REPOSITORY"] + meta + end + def extract_coverage return SimpleCov.result.covered_percent if defined?(SimpleCov) && SimpleCov.result 0.0 diff --git a/lizard.gemspec b/lizard.gemspec index 800b074..d1a79a9 100644 --- a/lizard.gemspec +++ b/lizard.gemspec @@ -30,5 +30,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rspec", "~> 3.0" spec.add_development_dependency "simplecov", "~> 0.22.0" spec.add_development_dependency "webmock", "~> 3.0" + spec.add_development_dependency "json_schemer", "~> 2.0" spec.add_development_dependency "mocha", "~> 3.0" end diff --git a/test/client_test.rb b/test/client_test.rb index b9ee087..0f25f42 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -29,6 +29,8 @@ def test_initialize_with_lizard_module_fallback end def test_send_test_run_returns_early_when_not_configured + ENV.delete("LIZARD_API_KEY") + ENV.delete("LIZARD_URL") client = Lizard::Client.new(api_key: nil, url: nil) result = client.send_test_run({test: "data"}) diff --git a/test/contract_test.rb b/test/contract_test.rb new file mode 100644 index 0000000..a22c018 --- /dev/null +++ b/test/contract_test.rb @@ -0,0 +1,135 @@ +require "test_helper" +require "net/http" +require "yaml" +require "json_schemer" + +class ContractTest < Minitest::Test + SPEC_URL = "https://djbender.github.io/lizard/openapi.yaml" + + def setup + @spec = fetch_spec + if @spec.nil? + msg = "OpenAPI spec not reachable at #{SPEC_URL}" + ENV["SKIP_CONTRACT_TESTS"] ? skip(msg) : raise(msg) + end + @schemas = @spec.dig("components", "schemas") + end + + def test_rspec_payload_matches_request_schema + payload = build_rspec_payload + schemer = schemer_for("CreateTestRunRequest") + + errors = schemer.validate(payload).to_a + assert_empty errors, "RSpec payload failed validation:\n#{format_errors(errors)}" + end + + def test_rspec_payload_with_metadata_matches_request_schema + payload = build_rspec_payload( + github_run_id: "23419710055", + github_repository: "djbender/lizard-ruby" + ) + schemer = schemer_for("CreateTestRunRequest") + + errors = schemer.validate(payload).to_a + assert_empty errors, "RSpec payload with metadata failed validation:\n#{format_errors(errors)}" + end + + def test_minitest_payload_matches_request_schema + payload = build_minitest_payload + schemer = schemer_for("CreateTestRunRequest") + + errors = schemer.validate(payload).to_a + assert_empty errors, "Minitest payload failed validation:\n#{format_errors(errors)}" + end + + def test_success_response_matches_schema + response = {"status" => "success", "id" => 1} + schemer = schemer_for("SuccessResponse") + + errors = schemer.validate(response).to_a + assert_empty errors, "Success response failed validation:\n#{format_errors(errors)}" + end + + def test_error_response_matches_schema + response = {"error" => "Invalid API key"} + schemer = schemer_for("ErrorResponse") + + errors = schemer.validate(response).to_a + assert_empty errors, "Error response failed validation:\n#{format_errors(errors)}" + end + + private + + def fetch_spec + uri = URI(SPEC_URL) + response = Net::HTTP.get_response(uri) + return nil unless response.is_a?(Net::HTTPSuccess) + YAML.safe_load(response.body) + rescue SocketError, Errno::ECONNREFUSED, Timeout::Error + nil + end + + def build_rspec_payload(github_run_id: nil, github_repository: nil) + metadata = {} + metadata["github_run_id"] = github_run_id if github_run_id + metadata["github_repository"] = github_repository if github_repository + + { + "test_run" => { + "commit_sha" => "abc123def456", + "branch" => "main", + "ruby_specs" => 42, + "js_specs" => 0, + "runtime" => 12.345, + "coverage" => 95.5, + "ran_at" => Time.now.iso8601, + "metadata" => metadata + } + } + end + + def build_minitest_payload(github_run_id: nil, github_repository: nil) + metadata = {} + metadata["github_run_id"] = github_run_id if github_run_id + metadata["github_repository"] = github_repository if github_repository + + { + "test_run" => { + "commit_sha" => "abc123def456", + "branch" => "main", + "ruby_specs" => 10, + "js_specs" => 0, + "runtime" => 3.21, + "coverage" => 88.0, + "ran_at" => Time.now.iso8601, + "metadata" => metadata + } + } + end + + def schemer_for(schema_name) + schema = @schemas.fetch(schema_name) + resolved = resolve_refs(schema) + JSONSchemer.schema(resolved) + end + + def resolve_refs(node) + case node + when Hash + if node.key?("$ref") + ref_name = node["$ref"].split("/").last + resolve_refs(@schemas.fetch(ref_name)) + else + node.transform_values { |v| resolve_refs(v) } + end + when Array + node.map { |v| resolve_refs(v) } + else + node + end + end + + def format_errors(errors) + errors.map { |e| " #{e["data_pointer"]}: #{e["type"]} — #{e["details"]}" }.join("\n") + end +end diff --git a/test/minitest_reporter_test.rb b/test/minitest_reporter_test.rb index 726bf90..b53c4da 100644 --- a/test/minitest_reporter_test.rb +++ b/test/minitest_reporter_test.rb @@ -1,7 +1,7 @@ require "test_helper" class MinitestReporterTest < Minitest::Test - LIZARD_ENV_KEYS = %w[LIZARD_TEST_MODE LIZARD_API_KEY LIZARD_URL LIZARD_REPORT].freeze + LIZARD_ENV_KEYS = %w[LIZARD_TEST_MODE LIZARD_API_KEY LIZARD_URL LIZARD_REPORT GITHUB_RUN_ID GITHUB_REPOSITORY].freeze def setup @original_env = LIZARD_ENV_KEYS.to_h { |k| [k, ENV[k]] } @@ -28,6 +28,7 @@ def test_callback_methods_exist end def test_report_sends_to_lizard_when_configured + ENV.delete("LIZARD_TEST_MODE") ENV["LIZARD_API_KEY"] = "test_key" ENV["LIZARD_URL"] = "https://test.example.com" ENV["LIZARD_REPORT"] = "true" @@ -49,7 +50,60 @@ def test_report_sends_to_lizard_when_configured end end + def test_report_includes_github_metadata_when_env_vars_present + ENV.delete("LIZARD_TEST_MODE") + ENV["LIZARD_API_KEY"] = "test_key" + ENV["LIZARD_URL"] = "https://test.example.com" + ENV["LIZARD_REPORT"] = "true" + ENV["GITHUB_RUN_ID"] = "23419710055" + ENV["GITHUB_REPOSITORY"] = "djbender/lizard-ruby" + + expected_metadata = {github_run_id: "23419710055", github_repository: "djbender/lizard-ruby"} + + client = mock + client.expects(:send_test_run).with(has_entry(:metadata, expected_metadata)) + + Lizard::Client.stubs(:new).returns(client) + + @reporter.stubs(:count).returns(5) + @reporter.stubs(:total_time).returns(1.5) + @reporter.stubs(:`).with("git rev-parse HEAD").returns("abc123") + @reporter.stubs(:`).with("git branch --show-current").returns("main") + + SimpleCov.stubs(:result).returns(mock(covered_percent: 85.0)) + + capture_io do + @reporter.report + end + end + + def test_report_sends_empty_metadata_without_github_env_vars + ENV.delete("LIZARD_TEST_MODE") + ENV.delete("GITHUB_RUN_ID") + ENV.delete("GITHUB_REPOSITORY") + ENV["LIZARD_API_KEY"] = "test_key" + ENV["LIZARD_URL"] = "https://test.example.com" + ENV["LIZARD_REPORT"] = "true" + + client = mock + client.expects(:send_test_run).with(has_entry(:metadata, {})) + + Lizard::Client.stubs(:new).returns(client) + + @reporter.stubs(:count).returns(5) + @reporter.stubs(:total_time).returns(1.5) + @reporter.stubs(:`).with("git rev-parse HEAD").returns("abc123") + @reporter.stubs(:`).with("git branch --show-current").returns("main") + + SimpleCov.stubs(:result).returns(mock(covered_percent: 85.0)) + + capture_io do + @reporter.report + end + end + def test_report_handles_nil_simplecov_result + ENV.delete("LIZARD_TEST_MODE") ENV["LIZARD_API_KEY"] = "test_key" ENV["LIZARD_URL"] = "https://test.example.com" ENV["LIZARD_REPORT"] = "true" diff --git a/test/rspec_formatter_test.rb b/test/rspec_formatter_test.rb index fffa3ba..d8dcc2d 100644 --- a/test/rspec_formatter_test.rb +++ b/test/rspec_formatter_test.rb @@ -1,7 +1,7 @@ require "test_helper" class RSpecFormatterTest < Minitest::Test - LIZARD_ENV_KEYS = %w[LIZARD_TEST_MODE LIZARD_API_KEY LIZARD_URL LIZARD_REPORT].freeze + LIZARD_ENV_KEYS = %w[LIZARD_TEST_MODE LIZARD_API_KEY LIZARD_URL LIZARD_REPORT GITHUB_RUN_ID GITHUB_REPOSITORY].freeze def setup @original_env = LIZARD_ENV_KEYS.to_h { |k| [k, ENV[k]] } @@ -28,6 +28,7 @@ def test_initialize_accepts_output_parameter end def test_dump_summary_sends_to_lizard_when_configured + ENV.delete("LIZARD_TEST_MODE") ENV["LIZARD_API_KEY"] = "test_key" ENV["LIZARD_URL"] = "https://test.example.com" ENV["LIZARD_REPORT"] = "true" @@ -49,7 +50,60 @@ def test_dump_summary_sends_to_lizard_when_configured @formatter.dump_summary(summary) end + def test_dump_summary_includes_github_metadata_when_env_vars_present + ENV.delete("LIZARD_TEST_MODE") + ENV["LIZARD_API_KEY"] = "test_key" + ENV["LIZARD_URL"] = "https://test.example.com" + ENV["LIZARD_REPORT"] = "true" + ENV["GITHUB_RUN_ID"] = "23419710055" + ENV["GITHUB_REPOSITORY"] = "djbender/lizard-ruby" + + expected_metadata = {github_run_id: "23419710055", github_repository: "djbender/lizard-ruby"} + + client = mock + client.expects(:send_test_run).with(has_entry(:metadata, expected_metadata)) + + Lizard::Client.stubs(:new).returns(client) + + @formatter.stubs(:`).with("git rev-parse HEAD").returns("abc123") + @formatter.stubs(:`).with("git branch --show-current").returns("main") + + summary = mock + summary.stubs(:example_count).returns(10) + summary.stubs(:duration).returns(2.5) + + SimpleCov.stubs(:result).returns(mock(covered_percent: 90.0)) + + @formatter.dump_summary(summary) + end + + def test_dump_summary_sends_empty_metadata_without_github_env_vars + ENV.delete("LIZARD_TEST_MODE") + ENV.delete("GITHUB_RUN_ID") + ENV.delete("GITHUB_REPOSITORY") + ENV["LIZARD_API_KEY"] = "test_key" + ENV["LIZARD_URL"] = "https://test.example.com" + ENV["LIZARD_REPORT"] = "true" + + client = mock + client.expects(:send_test_run).with(has_entry(:metadata, {})) + + Lizard::Client.stubs(:new).returns(client) + + @formatter.stubs(:`).with("git rev-parse HEAD").returns("abc123") + @formatter.stubs(:`).with("git branch --show-current").returns("main") + + summary = mock + summary.stubs(:example_count).returns(10) + summary.stubs(:duration).returns(2.5) + + SimpleCov.stubs(:result).returns(mock(covered_percent: 90.0)) + + @formatter.dump_summary(summary) + end + def test_dump_summary_handles_nil_simplecov_result + ENV.delete("LIZARD_TEST_MODE") ENV["LIZARD_API_KEY"] = "test_key" ENV["LIZARD_URL"] = "https://test.example.com" ENV["LIZARD_REPORT"] = "true" diff --git a/test/test_helper.rb b/test/test_helper.rb index 988b65c..4835bed 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -15,6 +15,8 @@ require "minitest/autorun" require "mocha/minitest" +ENV["LIZARD_TEST_MODE"] = "true" unless ENV["LIZARD_REPORT"] == "true" + Minitest.extensions << "lizard" module Minitest