diff --git a/app/tools/mcp_tools/failing_servers.rb b/app/tools/mcp_tools/failing_servers.rb index 5a56ff6c..5590f076 100644 --- a/app/tools/mcp_tools/failing_servers.rb +++ b/app/tools/mcp_tools/failing_servers.rb @@ -37,17 +37,23 @@ def self.call(query:, branch: nil, from: nil, to: nil, server_context: nil) end match_ids = matches.map(&:id).to_set + # Latest report id per configuration in one grouped query. A per-config + # ORDER BY datetime DESC LIMIT 1 loads wide rows without a usable index + # and takes seconds each on production. + latest_ids = Report.where(server_id: matches.map(&:server_id).uniq, branch: matches.map(&:branch).uniq). + where("datetime >= ?", from_t). + group(:server_id, :branch, :option).maximum(:id) + configs = matches.group_by { |r| [r.server_id, r.branch, r.option] }.map do |(server_id, br, opt), reports| first = reports.first last = reports.last - latest = Report.where(server_id: server_id, branch: br, option: opt).order(datetime: :desc).first { server: first.server.name, server_id: server_id, branch: br, option: opt, occurrences: reports.size, - still_failing: match_ids.include?(latest.id), + still_failing: match_ids.include?(latest_ids[[server_id, br, opt]]), first_seen: first.as_mcp_json, last_seen: last.as_mcp_json.merge(matching_line: last.log_excerpt&.matching_line(query)).compact, } diff --git a/app/tools/mcp_tools/find_failure_origin.rb b/app/tools/mcp_tools/find_failure_origin.rb index 366b072a..596f07c4 100644 --- a/app/tools/mcp_tools/find_failure_origin.rb +++ b/app/tools/mcp_tools/find_failure_origin.rb @@ -3,6 +3,16 @@ class FindFailureOrigin < MCP::Tool extend Helpers MAX_REPORTS_PER_CONFIG = 300 + MAX_MATCH_IDS = 10_000 + + # Light per-report tuple loaded via pluck. Loading full rows per + # configuration (ltsv is several KB each) took seconds per query on + # production and blew past the 30s request timeout. + Row = Struct.new(:id, :server_id, :option, :datetime, :summary) do + def success? + summary.include?(" success") + end + end tool_name "find_failure_origin" description "Locate when a failure started and the suspect commit range. For each failing " \ @@ -34,31 +44,45 @@ def self.call(branch:, server: nil, option: nil, query: nil, lookback_days: 90, end scope = scope.where(option: option) if option.present? - configs = scope.distinct.pluck(:server_id, :option) - return error_response("No reports found for branch #{branch} in the lookback window") if configs.empty? + rows = scope.pluck(:id, :server_id, :option, :datetime, :summary).map { |r| Row.new(*r) } + return error_response("No reports found for branch #{branch} in the lookback window") if rows.empty? match_ids = nil if query.present? - match_ids = scope.joins(:log_excerpt).merge(LogExcerpt.content_match(query)).pluck(:id).to_set + match_ids = scope.joins(:log_excerpt).merge(LogExcerpt.content_match(query)). + limit(MAX_MATCH_IDS).pluck(:id).to_set if match_ids.empty? return json_response(branch: branch, query: query, message: "No failure log matching the query in the lookback window") end end - results = configs.filter_map do |server_id, opt| - reports = Report.includes(:server). - where(branch: branch, server_id: server_id, option: opt). - where("datetime > ?", since). - order(datetime: :desc).limit(MAX_REPORTS_PER_CONFIG).to_a - next if reports.empty? || reports.first.server.nil? - query.present? ? analyze_query(reports, match_ids, query) : analyze_streak(reports) + boundaries = rows.group_by { |r| [r.server_id, r.option] }.filter_map do |_config, rs| + rs.sort_by!(&:datetime) + rs.reverse! + rs = rs.first(MAX_REPORTS_PER_CONFIG) + query.present? ? analyze_query(rs, match_ids) : analyze_streak(rs) end - if results.empty? + if boundaries.empty? return json_response(branch: branch, query: query, message: "No failing configuration found in the lookback window") end + ids = boundaries.flat_map { |b| [b[:first_bad_id], b[:last_good_id]] }.compact + records = Report.where(id: ids).includes(:server).index_by(&:id) + + results = boundaries.filter_map do |b| + first_bad = records[b[:first_bad_id]] + next if first_bad.nil? || first_bad.server.nil? + last_good = b[:last_good_id] && records[b[:last_good_id]] + entry = config_entry(first_bad, last_good, b) + if query.present? + line = first_bad.log_excerpt&.matching_line(query) + entry[:first_bad][:matching_line] = line if line + end + entry + end + results.sort_by! { |r| r[:first_bad][:datetime] } earliest = results.first json_response( @@ -77,36 +101,39 @@ def self.call(branch:, server: nil, option: nil, query: nil, lookback_days: 90, # Boundary of the current failing streak: skip configurations whose latest # report succeeded. - def self.analyze_streak(reports) - return nil if reports.first.success? - streak = reports.take_while { |r| !r.success? } - first_bad = streak.last - last_good = reports[streak.size] - config_entry(first_bad, last_good, reports, occurrences: streak.size) + def self.analyze_streak(rows) + return nil if rows.first.success? + streak = rows.take_while { |r| !r.success? } + { + first_bad_id: streak.last.id, + last_good_id: rows[streak.size]&.id, + occurrences: streak.size, + still_failing: true, + } end # Earliest report whose failure log matches the query. last_good is the # newest report older than it without the matching failure (it may have # failed for an unrelated reason). - def self.analyze_query(reports, match_ids, query) - matching = reports.select { |r| match_ids.include?(r.id) } + def self.analyze_query(rows, match_ids) + matching = rows.select { |r| match_ids.include?(r.id) } return nil if matching.empty? first_bad = matching.last - last_good = reports[reports.index(first_bad) + 1] - entry = config_entry(first_bad, last_good, reports, occurrences: matching.size, - still_failing: match_ids.include?(reports.first.id)) - line = first_bad.log_excerpt&.matching_line(query) - entry[:first_bad][:matching_line] = line if line - entry + { + first_bad_id: first_bad.id, + last_good_id: rows[rows.index(first_bad) + 1]&.id, + occurrences: matching.size, + still_failing: match_ids.include?(rows.first.id), + } end - def self.config_entry(first_bad, last_good, reports, occurrences:, still_failing: true) + def self.config_entry(first_bad, last_good, boundary) { server: first_bad.server.name, server_id: first_bad.server_id, option: first_bad.option, - still_failing: still_failing, - occurrences: occurrences, + still_failing: boundary[:still_failing], + occurrences: boundary[:occurrences], first_bad: first_bad.as_mcp_json, last_good: last_good&.as_mcp_json, compare_url: compare_url(last_good, first_bad),