From 5905ba4b005a19af70349ed56edcde31cb65ebce Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 09:49:30 +0900 Subject: [PATCH 01/11] Add log_excerpts with trigram search and S3 capture Store fail.txt and diff.txt excerpts of failed builds in Postgres so they can be searched with pg_trgm, since S3 offers no full-text search. Excerpts are capped at 100KB per report and captured right after scan_recent_ltsv creates a report; S3 errors are logged and skipped so they never break report ingestion. Co-Authored-By: Claude Fable 5 --- app/models/log_excerpt.rb | 67 ++++++++ app/models/report.rb | 25 ++- .../20260728023000_create_log_excerpts.rb | 18 +++ db/schema.rb | 15 +- test/models/log_excerpt_test.rb | 151 ++++++++++++++++++ 5 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 app/models/log_excerpt.rb create mode 100644 db/migrate/20260728023000_create_log_excerpts.rb create mode 100644 test/models/log_excerpt_test.rb diff --git a/app/models/log_excerpt.rb b/app/models/log_excerpt.rb new file mode 100644 index 00000000..5a09f560 --- /dev/null +++ b/app/models/log_excerpt.rb @@ -0,0 +1,67 @@ +require 'net/http' +require 'zlib' +require 'stringio' + +class LogExcerpt < ApplicationRecord + belongs_to :report + + MAX_CONTENT_BYTES = 100_000 + + # Fetch fail.txt and diff.txt from S3 for the report and store them as a + # searchable excerpt. Creates an empty-content row when nothing is available + # (missing or empty objects) so that backfill can resume by max report_id. + # Network errors are raised to the caller. + def self.capture(report) + return nil unless %r{\Ahttps?://rubyci\.s3\.amazonaws\.com/}.match?(report.server&.uri.to_s) + content = fetch_content(report) + excerpt = find_or_initialize_by(report_id: report.id) + excerpt.content = content + excerpt.save! + excerpt + end + + def self.fetch_content(report) + sections = [report.failtxt_uri, report.difftxt_uri].filter_map do |uri| + body = fetch_text(uri) + body unless body.nil? || body.empty? + end + truncate_bytes(sections.join("\n"), MAX_CONTENT_BYTES) + end + + # GET a gzipped text file. Returns nil on 404. S3 serves these either with + # Content-Encoding: gzip (Net::HTTP inflates the body) or as raw gzip bytes. + def self.fetch_text(uri) + uri = URI(uri) + res = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 30, use_ssl: uri.scheme == "https") do |h| + h.get(uri.path) + end + return nil if res.code == "404" + res.value + body = res.body + begin + body = Zlib::GzipReader.new(StringIO.new(body)).read + rescue Zlib::Error + end + body.force_encoding(Encoding::UTF_8).scrub("?").tr("\0", "") + end + + def self.truncate_bytes(str, max) + return str if str.bytesize <= max + str.byteslice(0, max).scrub("") + end + + # Substring match on content. On PostgreSQL the trigram GIN index makes + # ILIKE efficient; SQLite (development/test) falls back to plain LIKE. + def self.content_match(query) + operator = defined?(SQLite3) ? "LIKE" : "ILIKE" + # explicit ESCAPE: SQLite's LIKE has no default escape character + where("log_excerpts.content #{operator} ? ESCAPE '\\'", "%#{sanitize_sql_like(query)}%") + end + + def matching_line(query) + return nil if query.blank? + q = query.downcase + line = content.each_line.find { |l| l.downcase.include?(q) } + line&.strip&.truncate(200) + end +end diff --git a/app/models/report.rb b/app/models/report.rb index a6275c64..2aefd093 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -6,6 +6,7 @@ class Report < ApplicationRecord belongs_to :server + has_one :log_excerpt, dependent: :delete validates :server_id, :presence => true # SVN revision number or full 40-hex git commit SHA validates :revision, :format => { :with => /\A(?:\d+|\h{40})\z/ }, allow_nil: true @@ -130,6 +131,21 @@ def recenturi server.recent_uri(branch_opts) end + def failtxt_uri + s3txt_uri('compressed_failhtml_relpath', 'fail') + end + + def difftxt_uri + s3txt_uri('compressed_diffhtml_relpath', 'diff') + end + + def s3txt_uri(key, kind) + relpath = meta&.[](key)&.sub('.html.gz', '.txt.gz') || + datetime.strftime("log/%Y%m%dT%H%M%SZ.#{kind}.txt.gz") + "#{server.uri.chomp('/')}/#{depsuffixed_name}/#{relpath}" + end + private :s3txt_uri + def meta if defined?(@meta) @meta @@ -174,7 +190,7 @@ def self.scan_recent_ltsv(server, depsuffixed_name, body) diff = h["different_sections"] summary << (diff ? " (diff:#{diff})" : " (no diff)") - Report.create!( + report = Report.create!( server_id: server.id, datetime: datetime, branch: branch, @@ -183,6 +199,13 @@ def self.scan_recent_ltsv(server, depsuffixed_name, body) ltsv: line, summary: summary.gsub(/<[^>]*>/, ''), ) + if h["result"] != "success" + begin + LogExcerpt.capture(report) + rescue => e + warn [e, server.uri, "failed to capture log excerpt", report.id].inspect + end + end end rescue RuntimeError => e # It seems not a chkbuild log warn [e, server.uri, path, "failed to scan_reports"].inspect diff --git a/db/migrate/20260728023000_create_log_excerpts.rb b/db/migrate/20260728023000_create_log_excerpts.rb new file mode 100644 index 00000000..6c418e07 --- /dev/null +++ b/db/migrate/20260728023000_create_log_excerpts.rb @@ -0,0 +1,18 @@ +class CreateLogExcerpts < ActiveRecord::Migration[8.1] + def change + create_table :log_excerpts do |t| + t.integer :report_id, null: false + t.text :content, null: false + t.timestamps + t.index [:report_id], unique: true + end + add_foreign_key :log_excerpts, :reports + + # pg_trgm and GIN index are PostgreSQL-only; development and test run on SQLite + if connection.adapter_name == "PostgreSQL" + enable_extension "pg_trgm" + add_index :log_excerpts, :content, using: :gin, opclass: :gin_trgm_ops, + name: "index_log_excerpts_on_content_trgm" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 600b9eb8..af1dd98f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,10 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_07_27_095902) do +ActiveRecord::Schema[8.1].define(version: 2026_07_28_023000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" + enable_extension "pg_trgm" create_table "active_storage_attachments", force: :cascade do |t| t.bigint "blob_id", null: false @@ -42,6 +43,15 @@ t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end + create_table "log_excerpts", force: :cascade do |t| + t.text "content", null: false + t.datetime "created_at", null: false + t.integer "report_id", null: false + t.datetime "updated_at", null: false + t.index ["content"], name: "index_log_excerpts_on_content_trgm", opclass: :gin_trgm_ops, using: :gin + t.index ["report_id"], name: "index_log_excerpts_on_report_id", unique: true + end + create_table "recents", force: :cascade do |t| t.datetime "created_at", null: false t.string "etag", null: false @@ -63,7 +73,7 @@ t.datetime "updated_at", precision: nil t.index ["branch"], name: "index_reports_on_branch" t.index ["datetime"], name: "index_reports_on_datetime" - t.index ["revision"], name: "index_reports_on_revision", opclass: :varchar_pattern_ops + t.index ["revision"], name: "index_reports_on_revision" t.index ["server_id", "branch", "option"], name: "index_reports_on_server_id_and_branch_and_option" end @@ -79,5 +89,6 @@ add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "log_excerpts", "reports" add_foreign_key "recents", "servers" end diff --git a/test/models/log_excerpt_test.rb b/test/models/log_excerpt_test.rb new file mode 100644 index 00000000..6fa2b1c9 --- /dev/null +++ b/test/models/log_excerpt_test.rb @@ -0,0 +1,151 @@ +require "test_helper" + +class LogExcerptTest < ActiveSupport::TestCase + @@ordinal = 100 + + def create_server(name, uri: "https://rubyci.s3.amazonaws.com/#{name}/") + Server.create!(name: name, uri: uri, ordinal: (@@ordinal += 1)) + end + + # minitest 6 no longer bundles minitest/mock, so swap the singleton method + def stub_fetch_text(callable) + singleton = LogExcerpt.singleton_class + original = LogExcerpt.method(:fetch_text) + singleton.send(:define_method, :fetch_text) { |uri| callable.call(uri) } + yield + ensure + singleton.send(:define_method, :fetch_text, original) + end + + def create_report(server, attrs = {}) + Report.create!({ + server: server, + branch: "master", + datetime: Time.now.utc, + summary: "ruby 3.5.0dev failed(test-all)", + }.merge(attrs)) + end + + test "failtxt_uri and difftxt_uri from ltsv relpath" do + server = create_server("uri-ltsv") + report = create_report(server, ltsv: "depsuffixed_name:ruby-master\tcompressed_failhtml_relpath:log/20260727T023004Z.fail.html.gz\tcompressed_diffhtml_relpath:log/20260727T023004Z.diff.html.gz") + assert_equal "https://rubyci.s3.amazonaws.com/uri-ltsv/ruby-master/log/20260727T023004Z.fail.txt.gz", report.failtxt_uri + assert_equal "https://rubyci.s3.amazonaws.com/uri-ltsv/ruby-master/log/20260727T023004Z.diff.txt.gz", report.difftxt_uri + end + + test "failtxt_uri falls back to datetime" do + server = create_server("uri-dt") + report = create_report(server, datetime: Time.utc(2026, 7, 27, 2, 30, 4)) + assert_equal "https://rubyci.s3.amazonaws.com/uri-dt/master/log/20260727T023004Z.fail.txt.gz", report.failtxt_uri + end + + test "capture stores fail and diff content" do + server = create_server("cap-both") + report = create_report(server) + fetch = ->(uri) { uri.include?(".fail.") ? "FAIL BODY" : "DIFF BODY" } + stub_fetch_text(fetch) do + LogExcerpt.capture(report) + end + assert_equal "FAIL BODY\nDIFF BODY", report.reload.log_excerpt.content + end + + test "capture truncates content to MAX_CONTENT_BYTES" do + server = create_server("cap-trunc") + report = create_report(server) + stub_fetch_text(->(uri) { "x" * 200_000 }) do + LogExcerpt.capture(report) + end + assert_equal LogExcerpt::MAX_CONTENT_BYTES, report.reload.log_excerpt.content.bytesize + end + + test "capture saves empty row when nothing is available" do + server = create_server("cap-empty") + report = create_report(server) + stub_fetch_text(->(uri) { nil }) do + LogExcerpt.capture(report) + end + assert_equal "", report.reload.log_excerpt.content + end + + test "capture skips non-rubyci servers" do + server = create_server("cap-other", uri: "https://example.com/chkbuild/logs") + report = create_report(server) + stub_fetch_text(->(uri) { flunk "must not fetch" }) do + assert_nil LogExcerpt.capture(report) + end + assert_nil report.reload.log_excerpt + end + + test "capture is idempotent per report" do + server = create_server("cap-idem") + report = create_report(server) + stub_fetch_text(->(uri) { "first" }) do + LogExcerpt.capture(report) + end + stub_fetch_text(->(uri) { "second" }) do + LogExcerpt.capture(report) + end + assert_equal 1, LogExcerpt.where(report_id: report.id).count + assert_equal "second\nsecond", report.reload.log_excerpt.content + end + + test "content_match finds substring" do + server = create_server("cap-match") + report = create_report(server) + LogExcerpt.create!(report: report, content: "TestFoo#test_bar [test.rb:1]:\nExpected 1 to equal 2.") + assert_includes LogExcerpt.content_match("test_bar").to_a.map(&:report_id), report.id + assert_empty LogExcerpt.content_match("no_such_token").where(report_id: report.id).to_a + end + + test "matching_line returns first matching line" do + excerpt = LogExcerpt.new(content: "line one\nTestFoo#test_bar failed\nline three") + assert_equal "TestFoo#test_bar failed", excerpt.matching_line("TEST_BAR") + assert_nil excerpt.matching_line("absent") + assert_nil excerpt.matching_line("") + end + + test "scan_recent_ltsv captures excerpt for failed build" do + server = create_server("scan-fail") + line = [ + "start_time:20260727T051800Z", + "title:ruby 3.5.0dev (2026-07-27) [x86_64-linux]", + "result:failure", + "compressed_failhtml_relpath:log/20260727T051800Z.fail.html.gz", + "compressed_diffhtml_relpath:log/20260727T051800Z.diff.html.gz", + ].join("\t") + stub_fetch_text(->(uri) { "some failure output" }) do + Report.scan_recent_ltsv(server, "ruby-master", line + "\n") + end + report = Report.where(server_id: server.id).last + assert_equal "some failure output\nsome failure output", report.log_excerpt.content + end + + test "scan_recent_ltsv does not capture excerpt for success build" do + server = create_server("scan-ok") + line = [ + "start_time:20260727T051800Z", + "title:ruby 3.5.0dev (2026-07-27) [x86_64-linux]", + "result:success", + ].join("\t") + stub_fetch_text(->(uri) { flunk "must not fetch" }) do + Report.scan_recent_ltsv(server, "ruby-master", line + "\n") + end + report = Report.where(server_id: server.id).last + assert_nil report.log_excerpt + end + + test "scan_recent_ltsv survives capture errors" do + server = create_server("scan-err") + line = [ + "start_time:20260727T051800Z", + "title:ruby 3.5.0dev (2026-07-27) [x86_64-linux]", + "result:failure", + ].join("\t") + stub_fetch_text(->(uri) { raise Net::OpenTimeout }) do + Report.scan_recent_ltsv(server, "ruby-master", line + "\n") + end + report = Report.where(server_id: server.id).last + assert_not_nil report + assert_nil report.log_excerpt + end +end From 63ffff931d349879bb101f4630f35f4bc0e22273 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 09:49:39 +0900 Subject: [PATCH 02/11] Add backfill and prune tasks for log excerpts log_excerpts:backfill fills excerpts for past failed builds (default last 1 year, FROM/TO to override) and resumes from the max stored report_id when interrupted. log_excerpts:prune deletes excerpts older than KEEP_DAYS for Heroku Scheduler. Co-Authored-By: Claude Fable 5 --- lib/tasks/log_excerpts.rake | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 lib/tasks/log_excerpts.rake diff --git a/lib/tasks/log_excerpts.rake b/lib/tasks/log_excerpts.rake new file mode 100644 index 00000000..8d9c7843 --- /dev/null +++ b/lib/tasks/log_excerpts.rake @@ -0,0 +1,46 @@ +namespace :log_excerpts do + desc "Backfill log excerpts from S3. FROM/TO (date, default last 1 year), START_ID to force resume point, SLEEP (default 0.1)" + task :backfill => :environment do + from = ENV["FROM"] ? Time.parse(ENV["FROM"]) : 1.year.ago + to = ENV["TO"] ? Time.parse(ENV["TO"]) : Time.now + interval = (ENV["SLEEP"] || 0.1).to_f + + scope = Report.joins(:server). + where(datetime: from..to). + where.not(ltsv: nil). + where("ltsv NOT LIKE '%result:success%'"). + where("servers.uri LIKE 'https://rubyci.s3.amazonaws.com/%'") + + start_id = ENV["START_ID"]&.to_i || + LogExcerpt.joins(:report).where(reports: { datetime: from..to }).maximum(:report_id) || + 0 + remaining = scope.where("reports.id > ?", start_id) + total = remaining.count + puts "backfilling #{total} reports (datetime #{from}..#{to}, report_id > #{start_id})" + + done = 0 + remaining.includes(:server).find_each(batch_size: 100) do |report| + begin + LogExcerpt.capture(report) + rescue => e + warn "report #{report.id}: #{e.class}: #{e.message}" + end + done += 1 + puts "#{done}/#{total} (report_id=#{report.id})" if done % 100 == 0 + sleep interval + end + puts "done: #{done}/#{total}" + end + + desc "Delete log excerpts older than KEEP_DAYS (default 366) days" + task :prune => :environment do + keep_days = (ENV["KEEP_DAYS"] || 366).to_i + cutoff = keep_days.days.ago + scope = LogExcerpt.where(report_id: Report.where("datetime < ?", cutoff).select(:id)) + deleted = 0 + scope.in_batches(of: 10_000) do |batch| + deleted += batch.delete_all + end + puts "deleted #{deleted} log excerpts older than #{cutoff}" + end +end From 2ea9a32388e32ac6cf4915f4edb06dd622c880af Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 09:49:45 +0900 Subject: [PATCH 03/11] Add /search UI for failure log excerpts Search failed builds by excerpt text, revision prefix, server, branch and date range. Reports are narrowed by existing indexes before the trigram match, and results are paginated at 50 rows to stay within the Heroku router timeout. Co-Authored-By: Claude Fable 5 --- app/controllers/search_controller.rb | 50 +++++++++++++++ app/views/layouts/application.html.erb | 1 + app/views/search/index.html.erb | 74 ++++++++++++++++++++++ config/routes.rb | 2 + test/controllers/search_controller_test.rb | 61 ++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 app/controllers/search_controller.rb create mode 100644 app/views/search/index.html.erb create mode 100644 test/controllers/search_controller_test.rb diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb new file mode 100644 index 00000000..a142d5da --- /dev/null +++ b/app/controllers/search_controller.rb @@ -0,0 +1,50 @@ +class SearchController < ApplicationController + PER_PAGE = 50 + + # GET /search + # Searches log excerpts of failed builds. Reports are narrowed by the + # indexed columns (server_id, branch, datetime, revision) first, then + # matched against log_excerpts.content (trigram GIN index on PostgreSQL). + def index + @q = params[:q].to_s.strip + @server_id = params[:server].to_s[/\A\d+\z/] + @branch = params[:branch].to_s.strip + @revision = params[:revision].to_s.strip + @from = parse_time(params[:from]) + @to = parse_time(params[:to], end_of_day: true) + @page = [params[:page].to_i, 1].max + @servers = Server.order(:ordinal).to_a + + @searched = @q.present? || @revision.present? || @branch.present? || + @server_id.present? || @from || @to + unless @searched + @reports = [] + return + end + + reports = Report.joins(:log_excerpt).includes(:server, :log_excerpt). + order("reports.datetime DESC") + reports = reports.where(server_id: @server_id) if @server_id + reports = reports.where(branch: @branch) if @branch.present? + reports = reports.where("reports.datetime >= ?", @from) if @from + reports = reports.where("reports.datetime <= ?", @to) if @to + if @revision.present? + reports = reports.where("reports.revision LIKE ?", "#{Report.sanitize_sql_like(@revision)}%") + end + reports = reports.merge(LogExcerpt.content_match(@q)) if @q.present? + + page = reports.limit(PER_PAGE + 1).offset((@page - 1) * PER_PAGE).to_a + @has_next = page.size > PER_PAGE + @reports = page.first(PER_PAGE) + end + + private + + def parse_time(str, end_of_day: false) + return nil if str.blank? + date = Date.parse(str) rescue nil + return nil unless date + time = Time.utc(date.year, date.month, date.day) + end_of_day ? time + 86399 : time + end +end diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 2bc968d7..b7782760 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -19,6 +19,7 @@
  • <%= link_to "Current Reports", "/reports", class: "nav-link" %>
  • <%= link_to "Latest Reports", "/", class: "nav-link" %>
    • +
  • <%= link_to "Search", "/search", class: "nav-link" %>
diff --git a/app/views/search/index.html.erb b/app/views/search/index.html.erb new file mode 100644 index 00000000..b3cf84e9 --- /dev/null +++ b/app/views/search/index.html.erb @@ -0,0 +1,74 @@ +<% content_for(:title) { "Search failures - RubyCI" } %> + +

Search failure logs

+ +<%= form_with url: search_path, method: :get, local: true do %> +

+ + +

+

+ + + + + <%= submit_tag "Search" %> +

+<% end %> + +<% if @searched %> + <% if @reports.empty? %> +

No matching failure logs found<%= " on page #{@page}" if @page > 1 %>.

+ <% else %> +
+ + + + + + + + + + + + + + + <% @reports.each do |report| %> + <% + failuri = report.failuri || report.loguri + if report.revision&.match?(/\A\d+\z/) + revision = "r#{report.revision}" + elsif report.sha1 + revision = report.sha1[0, 11] + end + revision_link = link_to revision, report.revisionuri if revision + snippet = report.log_excerpt&.matching_line(@q) + %> + + + + + + + + + + + <% end %> + +
ServerDatetimeBranchOptionRevisionSummaryMatchDiff
<%= report.server.name %><%= link_to report.sjstdt, report.loguri, title: report.jstdt %><%= report.branch %><%= report.option %><%= revision_link %>
<%= link_to report.shortsummary, failuri, title: report.shortsummary %>
<%= snippet %>
<%= link_to report.diffstat, report.diffuri, title: report.diffstat %>
+
+

+ <% if @page > 1 %> + <%= link_to "Prev", search_path(request.query_parameters.merge("page" => @page - 1)) %> + <% end %> + <% if @has_next %> + <%= link_to "Next", search_path(request.query_parameters.merge("page" => @page + 1)) %> + <% end %> +

+ <% end %> +<% else %> +

Search the excerpted fail/diff logs of failed builds. Datetimes are UTC.

+<% end %> diff --git a/config/routes.rb b/config/routes.rb index 018709d0..15bc4a4a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,8 @@ Rails.application.routes.draw do root :to => 'reports#current' + get "search" => "search#index" + resources :reports, only: [:show, :index] do collection do get "current" diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb new file mode 100644 index 00000000..7b9d647a --- /dev/null +++ b/test/controllers/search_controller_test.rb @@ -0,0 +1,61 @@ +require "test_helper" + +class SearchControllerTest < ActionDispatch::IntegrationTest + SHA = "abcdef0123456789abcdef0123456789abcdef01" + + setup do + @server = Server.create!(name: "search-srv", uri: "https://rubyci.s3.amazonaws.com/search-srv/", ordinal: 999) + @report = Report.create!( + server: @server, + branch: "master", + datetime: Time.utc(2026, 7, 20, 1, 2, 3), + revision: SHA, + summary: "ruby 3.5.0dev failed(test-all) (diff:test-all)", + ltsv: "depsuffixed_name:ruby-master\tresult:failure", + ) + LogExcerpt.create!(report: @report, content: "TestSearch#test_hit [foo_test.rb:12]:\nExpected true.") + end + + test "top page without params shows only the form" do + get search_url + assert_response :success + assert_no_match "TestSearch#test_hit", response.body + end + + test "search by text matches excerpt content" do + get search_url, params: { q: "test_hit" } + assert_response :success + assert_match "TestSearch#test_hit", response.body + assert_match @report.loguri, response.body + end + + test "search by text with no match" do + get search_url, params: { q: "does_not_exist_anywhere" } + assert_response :success + assert_match "No matching failure logs", response.body + end + + test "search by revision prefix" do + get search_url, params: { revision: SHA[0, 10] } + assert_response :success + assert_match @report.loguri, response.body + end + + test "search narrowed by branch and date range" do + get search_url, params: { q: "test_hit", branch: "master", from: "2026-07-20", to: "2026-07-20" } + assert_response :success + assert_match @report.loguri, response.body + + get search_url, params: { q: "test_hit", branch: "ruby_3_4" } + assert_match "No matching failure logs", response.body + + get search_url, params: { q: "test_hit", to: "2026-07-19" } + assert_match "No matching failure logs", response.body + end + + test "search escapes LIKE wildcards" do + get search_url, params: { q: "%" } + assert_response :success + assert_match "No matching failure logs", response.body + end +end From 1a3d93ea80b6d0af6d58e023b5cfcff505cacac6 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 09:54:57 +0900 Subject: [PATCH 04/11] Hide revision field from the search form Keep the revision query parameter for machine clients (a planned MCP server needs to check whether reports exist for a commit), but drop the visible input since it is not useful for humans. Co-Authored-By: Claude Fable 5 --- app/views/search/index.html.erb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/views/search/index.html.erb b/app/views/search/index.html.erb index b3cf84e9..e01e5083 100644 --- a/app/views/search/index.html.erb +++ b/app/views/search/index.html.erb @@ -5,7 +5,9 @@ <%= form_with url: search_path, method: :get, local: true do %>

- + <%# The revision query parameter is kept for machine clients (planned MCP + server) but intentionally has no visible form field. %> + <%= hidden_field_tag :revision, @revision if @revision.present? %>

From cde17b33fc9f4c72ad6af62755658d01c649f596 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 09:57:26 +0900 Subject: [PATCH 05/11] Widen the search text field Co-Authored-By: Claude Fable 5 --- app/views/search/index.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/search/index.html.erb b/app/views/search/index.html.erb index e01e5083..df47acdb 100644 --- a/app/views/search/index.html.erb +++ b/app/views/search/index.html.erb @@ -4,7 +4,7 @@ <%= form_with url: search_path, method: :get, local: true do %>

- + <%# The revision query parameter is kept for machine clients (planned MCP server) but intentionally has no visible form field. %> <%= hidden_field_tag :revision, @revision if @revision.present? %> From 8e61a6bab6ca31dbd777fcccc79d4b22b30986d6 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:00:47 +0900 Subject: [PATCH 06/11] Align search form rows with flexbox Put the submit button right of the text field and spread the filter row so both rows share the same right edge. Co-Authored-By: Claude Fable 5 --- app/assets/stylesheets/application.css | 31 ++++++++++++++++++++++++++ app/views/search/index.html.erb | 10 ++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/app/assets/stylesheets/application.css b/app/assets/stylesheets/application.css index 83546616..f53c19be 100644 --- a/app/assets/stylesheets/application.css +++ b/app/assets/stylesheets/application.css @@ -237,3 +237,34 @@ table.reports td.diff div { text-overflow: ellipsis; width: 100px; } + +/* Search form */ + +form.search-form { + max-width: 55em; +} + +form.search-form .search-row { + display: flex; + align-items: center; + gap: 0.5em; + margin: 0.5em 0; +} + +form.search-form .search-row label { + display: flex; + align-items: center; + gap: 0.25em; +} + +form.search-form .search-text-row label { + flex: 1; +} + +form.search-form .search-text-row input[type="text"] { + flex: 1; +} + +form.search-form .search-filter-row { + justify-content: space-between; +} diff --git a/app/views/search/index.html.erb b/app/views/search/index.html.erb index df47acdb..09101957 100644 --- a/app/views/search/index.html.erb +++ b/app/views/search/index.html.erb @@ -2,19 +2,19 @@

Search failure logs

-<%= form_with url: search_path, method: :get, local: true do %> -

- +<%= form_with url: search_path, method: :get, local: true, html: { class: "search-form" } do %> +

+ <%# The revision query parameter is kept for machine clients (planned MCP server) but intentionally has no visible form field. %> <%= hidden_field_tag :revision, @revision if @revision.present? %> + <%= submit_tag "Search" %>

-

+

- <%= submit_tag "Search" %>

<% end %> From e8a870d0135a0adb3aba7521740c2e6413627645 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:43:36 +0900 Subject: [PATCH 07/11] Fix backfill resuming past every pending report The resume point came from the highest already-captured report_id, but scan_recent_ltsv captures new failures as they arrive, so that id is always the newest report and the task skipped everything behind it. It printed "backfilling 0 reports" on every run after the first live capture, which is exactly when the past year still needed importing. Select reports that have no excerpt instead, so an interrupted run resumes and a report whose fetch failed is retried. FORCE=1 refetches reports that already have one. Co-Authored-By: Claude Fable 5 --- lib/tasks/log_excerpts.rake | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/tasks/log_excerpts.rake b/lib/tasks/log_excerpts.rake index 8d9c7843..16c51d1c 100644 --- a/lib/tasks/log_excerpts.rake +++ b/lib/tasks/log_excerpts.rake @@ -1,9 +1,10 @@ namespace :log_excerpts do - desc "Backfill log excerpts from S3. FROM/TO (date, default last 1 year), START_ID to force resume point, SLEEP (default 0.1)" + desc "Backfill log excerpts from S3. FROM/TO (date, default last 1 year), START_ID to force resume point, SLEEP (default 0.1), FORCE=1 to refetch reports that already have an excerpt" task :backfill => :environment do from = ENV["FROM"] ? Time.parse(ENV["FROM"]) : 1.year.ago to = ENV["TO"] ? Time.parse(ENV["TO"]) : Time.now interval = (ENV["SLEEP"] || 0.1).to_f + start_id = ENV["START_ID"]&.to_i scope = Report.joins(:server). where(datetime: from..to). @@ -11,15 +12,16 @@ namespace :log_excerpts do where("ltsv NOT LIKE '%result:success%'"). where("servers.uri LIKE 'https://rubyci.s3.amazonaws.com/%'") - start_id = ENV["START_ID"]&.to_i || - LogExcerpt.joins(:report).where(reports: { datetime: from..to }).maximum(:report_id) || - 0 - remaining = scope.where("reports.id > ?", start_id) - total = remaining.count - puts "backfilling #{total} reports (datetime #{from}..#{to}, report_id > #{start_id})" + # Resume by skipping reports that already have an excerpt. Using the max + # captured report_id instead would skip everything, because scan_recent_ltsv + # captures new failures as they arrive and that id is always the newest one. + scope = scope.where.missing(:log_excerpt) unless ENV["FORCE"] + scope = scope.where("reports.id > ?", start_id) if start_id + total = scope.count + puts "backfilling #{total} reports (datetime #{from}..#{to})" done = 0 - remaining.includes(:server).find_each(batch_size: 100) do |report| + scope.includes(:server).find_each(batch_size: 100) do |report| begin LogExcerpt.capture(report) rescue => e From 9bb7c5e7494ba108063cd8dd5b649ad63ecaf3e6 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:43:55 +0900 Subject: [PATCH 08/11] Validate rake task arguments and report backfill outcome KEEP_DAYS= or KEEP_DAYS=abc took the empty-string branch and fell to 0 via to_i, which made prune delete every excerpt. FROM and TO parsed in the local zone while reports.datetime is UTC, and an unparseable date raised a bare backtrace. Every option now aborts with a message naming the offending value. Backfill counted loop iterations, so a run where every fetch raised still finished "done: N/N" and exited 0. Count captured, empty and failed separately and exit nonzero when anything failed. Add DRY_RUN to prune, and report progress from both tasks. Co-Authored-By: Claude Fable 5 --- lib/tasks/log_excerpts.rake | 75 +++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/lib/tasks/log_excerpts.rake b/lib/tasks/log_excerpts.rake index 16c51d1c..1f177a08 100644 --- a/lib/tasks/log_excerpts.rake +++ b/lib/tasks/log_excerpts.rake @@ -1,10 +1,35 @@ +module LogExcerptTaskEnv + module_function + + # UTC to match reports.datetime, which Report.scan_recent_ltsv stores as UTC. + def time(key, default) + value = ENV[key].presence or return default + date = begin + Date.strptime(value, "%Y-%m-%d") + rescue Date::Error + abort "#{key} must be a date like 2026-07-20 (UTC), got #{value.inspect}" + end + Time.utc(date.year, date.month, date.day) + end + + def integer(key, default) + value = ENV[key].presence or return default + Integer(value) rescue abort("#{key} must be an integer, got #{value.inspect}") + end + + def float(key, default) + value = ENV[key].presence or return default + Float(value) rescue abort("#{key} must be a number of seconds, got #{value.inspect}") + end +end + namespace :log_excerpts do - desc "Backfill log excerpts from S3. FROM/TO (date, default last 1 year), START_ID to force resume point, SLEEP (default 0.1), FORCE=1 to refetch reports that already have an excerpt" + desc "Backfill log excerpts from S3. FROM/TO (UTC date, default last 1 year), START_ID to force resume point, SLEEP seconds between reports (default 0.1), FORCE=1 to refetch reports that already have an excerpt" task :backfill => :environment do - from = ENV["FROM"] ? Time.parse(ENV["FROM"]) : 1.year.ago - to = ENV["TO"] ? Time.parse(ENV["TO"]) : Time.now - interval = (ENV["SLEEP"] || 0.1).to_f - start_id = ENV["START_ID"]&.to_i + from = LogExcerptTaskEnv.time("FROM", 1.year.ago) + to = LogExcerptTaskEnv.time("TO", Time.now) + interval = LogExcerptTaskEnv.float("SLEEP", 0.1) + start_id = LogExcerptTaskEnv.integer("START_ID", nil) scope = Report.joins(:server). where(datetime: from..to). @@ -17,32 +42,52 @@ namespace :log_excerpts do # captures new failures as they arrive and that id is always the newest one. scope = scope.where.missing(:log_excerpt) unless ENV["FORCE"] scope = scope.where("reports.id > ?", start_id) if start_id + total = scope.count - puts "backfilling #{total} reports (datetime #{from}..#{to})" + puts "backfilling #{total} reports (datetime #{from}..#{to}, sleep #{interval}s#{start_id ? ", report_id > #{start_id}" : ""}#{ENV["FORCE"] ? ", forced" : ""})" - done = 0 + started = Time.now + captured = empty = failed = 0 + failed_ids = [] scope.includes(:server).find_each(batch_size: 100) do |report| begin - LogExcerpt.capture(report) + excerpt = LogExcerpt.capture(report) + excerpt&.content.presence ? captured += 1 : empty += 1 rescue => e - warn "report #{report.id}: #{e.class}: #{e.message}" + failed += 1 + failed_ids << report.id + warn "report #{report.id} (#{report.failtxt_uri}): #{e.class}: #{e.message}" end - done += 1 - puts "#{done}/#{total} (report_id=#{report.id})" if done % 100 == 0 + done = captured + empty + failed + puts "#{done}/#{total} (report_id=#{report.id}, #{(Time.now - started).round}s elapsed)" if done % 100 == 0 sleep interval end - puts "done: #{done}/#{total}" + + puts "done in #{(Time.now - started).round}s: #{captured} captured, #{empty} empty, #{failed} failed" + unless failed.zero? + warn "failed report ids: #{failed_ids.join(", ")}" + abort "backfill finished with #{failed} failures" + end end - desc "Delete log excerpts older than KEEP_DAYS (default 366) days" + desc "Delete log excerpts whose report datetime is older than KEEP_DAYS (default 366) days. DRY_RUN=1 to only count." task :prune => :environment do - keep_days = (ENV["KEEP_DAYS"] || 366).to_i + keep_days = LogExcerptTaskEnv.integer("KEEP_DAYS", 366) + abort "KEEP_DAYS must be at least 1, got #{keep_days}" if keep_days < 1 cutoff = keep_days.days.ago scope = LogExcerpt.where(report_id: Report.where("datetime < ?", cutoff).select(:id)) + + if ENV["DRY_RUN"] + puts "would delete #{scope.count} log excerpts of reports older than #{cutoff}" + next + end + deleted = 0 scope.in_batches(of: 10_000) do |batch| deleted += batch.delete_all + puts "deleted #{deleted} ..." end - puts "deleted #{deleted} log excerpts older than #{cutoff}" + puts "deleted #{deleted} log excerpts of reports older than #{cutoff}" + puts "note: disk space returns via autovacuum; VACUUM FULL is needed to reclaim it immediately" end end From 828f767bd41dd8c7dec3ec4cb761597bb47d838c Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:44:04 +0900 Subject: [PATCH 09/11] Budget fail and diff excerpts separately The two sections were joined and then truncated as a whole, so a fail.txt over the limit left no room for the diff at all. Six of 25 sampled reports hit the cap, meaning a quarter of them were searchable only through their fail output. Give each section an equal share and let one donate its unused share to the other, so the common case where one side is small is unaffected. Co-Authored-By: Claude Fable 5 --- app/models/log_excerpt.rb | 9 ++++++++- test/models/log_excerpt_test.rb | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/app/models/log_excerpt.rb b/app/models/log_excerpt.rb index 5a09f560..34a24153 100644 --- a/app/models/log_excerpt.rb +++ b/app/models/log_excerpt.rb @@ -25,7 +25,14 @@ def self.fetch_content(report) body = fetch_text(uri) body unless body.nil? || body.empty? end - truncate_bytes(sections.join("\n"), MAX_CONTENT_BYTES) + return "" if sections.empty? + # Budget each section separately, otherwise a fail.txt over the limit + # crowds the diff out entirely. A section under budget donates its + # remainder to the other one. The separators come out of the budget so + # the joined result still fits in MAX_CONTENT_BYTES. + budget = (MAX_CONTENT_BYTES - (sections.size - 1)) / sections.size + slack = sections.sum { |s| [budget - s.bytesize, 0].max } + sections.map { |s| truncate_bytes(s, budget + slack) }.join("\n") end # GET a gzipped text file. Returns nil on 404. S3 serves these either with diff --git a/test/models/log_excerpt_test.rb b/test/models/log_excerpt_test.rb index 6fa2b1c9..9fc9ca24 100644 --- a/test/models/log_excerpt_test.rb +++ b/test/models/log_excerpt_test.rb @@ -55,7 +55,32 @@ def create_report(server, attrs = {}) stub_fetch_text(->(uri) { "x" * 200_000 }) do LogExcerpt.capture(report) end - assert_equal LogExcerpt::MAX_CONTENT_BYTES, report.reload.log_excerpt.content.bytesize + assert_operator report.reload.log_excerpt.content.bytesize, :<=, LogExcerpt::MAX_CONTENT_BYTES + end + + test "capture keeps the diff section when fail is over budget" do + server = create_server("cap-budget") + report = create_report(server) + fetch = ->(uri) { uri.include?(".fail.") ? "F" * 200_000 : "D" * 1_000 } + stub_fetch_text(fetch) do + LogExcerpt.capture(report) + end + content = report.reload.log_excerpt.content + assert_operator content.bytesize, :<=, LogExcerpt::MAX_CONTENT_BYTES + assert_equal 1_000, content.count("D") + assert_operator content.count("F"), :>, 90_000 + end + + test "capture lets one section use the other's unused budget" do + server = create_server("cap-slack") + report = create_report(server) + fetch = ->(uri) { uri.include?(".fail.") ? "F" * 90_000 : "D" * 1_000 } + stub_fetch_text(fetch) do + LogExcerpt.capture(report) + end + content = report.reload.log_excerpt.content + assert_equal 90_000, content.count("F") + assert_equal 1_000, content.count("D") end test "capture saves empty row when nothing is available" do From 012973e4047cd704ceb644595987d60dc45e9b49 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:44:11 +0900 Subject: [PATCH 10/11] Stop page param from raising on non-scalar and huge values ?page[]=1 made params[:page] an Array, which has no to_i, so the request reached the browser as a 500. A page number past the bigint range produced an OFFSET PostgreSQL rejects, with the same result. Co-Authored-By: Claude Fable 5 --- app/controllers/search_controller.rb | 5 ++++- test/controllers/search_controller_test.rb | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index a142d5da..d77c850d 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -1,5 +1,6 @@ class SearchController < ApplicationController PER_PAGE = 50 + MAX_PAGE = 200 # GET /search # Searches log excerpts of failed builds. Reports are narrowed by the @@ -12,7 +13,9 @@ def index @revision = params[:revision].to_s.strip @from = parse_time(params[:from]) @to = parse_time(params[:to], end_of_day: true) - @page = [params[:page].to_i, 1].max + # to_s first: params[:page] is an Array for ?page[]=1, which has no to_i. + # Clamping also keeps OFFSET within range for absurd page numbers. + @page = params[:page].to_s.to_i.clamp(1, MAX_PAGE) @servers = Server.order(:ordinal).to_a @searched = @q.present? || @revision.present? || @branch.present? || diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb index 7b9d647a..4fb2b7cb 100644 --- a/test/controllers/search_controller_test.rb +++ b/test/controllers/search_controller_test.rb @@ -53,6 +53,17 @@ class SearchControllerTest < ActionDispatch::IntegrationTest assert_match "No matching failure logs", response.body end + test "array page param does not raise" do + get search_url, params: { q: "test_hit", page: ["1"] } + assert_response :success + end + + test "out of range page is clamped" do + get search_url, params: { q: "test_hit", page: "99999999999999999999" } + assert_response :success + assert_match "No matching failure logs", response.body + end + test "search escapes LIKE wildcards" do get search_url, params: { q: "%" } assert_response :success From 32f2d6e33dd5374853899679a82087844edb259b Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 28 Jul 2026 10:44:19 +0900 Subject: [PATCH 11/11] Stop search results painting Match and Diff as failures The results table reuses table.reports, whose failure colour targets columns 6 through 9. Search has a different column order, so Match and Diff came out red on every row while meaning nothing. Scope the table with its own class and clear those two, leaving Summary red. Also give the Match snippet a width and a title, matching how Summary and Diff already truncate. Co-Authored-By: Claude Fable 5 --- app/assets/stylesheets/application.css | 15 +++++++++++++++ app/views/search/index.html.erb | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/assets/stylesheets/application.css b/app/assets/stylesheets/application.css index f53c19be..7e6f107b 100644 --- a/app/assets/stylesheets/application.css +++ b/app/assets/stylesheets/application.css @@ -268,3 +268,18 @@ form.search-form .search-text-row input[type="text"] { form.search-form .search-filter-row { justify-content: space-between; } + +/* Search results reuse table.reports, but their columns are in a different + order, so the reports failure-column rule lands on Match and Diff. */ +table.search-results td.match, +table.search-results td.diff { + background-color: transparent; +} + +table.search-results td.match code { + display: block; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + width: 320px; +} diff --git a/app/views/search/index.html.erb b/app/views/search/index.html.erb index 09101957..84f20868 100644 --- a/app/views/search/index.html.erb +++ b/app/views/search/index.html.erb @@ -23,7 +23,7 @@

No matching failure logs found<%= " on page #{@page}" if @page > 1 %>.

<% else %>
- +
@@ -55,7 +55,7 @@ - + <% end %>
Server<%= report.option %> <%= revision_link %>
<%= link_to report.shortsummary, failuri, title: report.shortsummary %>
<%= snippet %><%= snippet %>
<%= link_to report.diffstat, report.diffuri, title: report.diffstat %>