From 0710fa08c346bfa950931c46db1abfad0f2371b3 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 12:21:42 +0200 Subject: [PATCH 1/7] feat: add `stderr-logs` feature. (#716) If the `stderr-logs` feature is enabled the YARA-X library emits log messages to stderr. The CLI now uses the same feature for enabling logs instead of using `logging`. --- Cargo.lock | 3 +-- capi/Cargo.toml | 17 +++++++++++++++++ cli/Cargo.toml | 15 +++++++++------ cli/src/main.rs | 3 --- cli/src/walk.rs | 8 ++++---- lib/Cargo.toml | 33 ++++++++++++++++++++++++++++++--- lib/src/compiler/mod.rs | 2 ++ lib/src/compiler/rules.rs | 4 +++- lib/src/lib.rs | 30 ++++++++++++++++++++++++++++++ lib/src/scanner/context.rs | 4 +++- 10 files changed, 99 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cbf740f7..85442f2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4407,6 +4407,7 @@ dependencies = [ "dsa", "ecdsa", "encoding_rs", + "env_logger", "flate2", "getrandom 0.2.17", "globwalk", @@ -4490,13 +4491,11 @@ dependencies = [ "dunce", "enable-ansi-support", "encoding_rs", - "env_logger", "figment", "globwalk", "home", "indicatif", "itertools 0.14.0", - "log", "predicates", "protobuf", "regex", diff --git a/capi/Cargo.toml b/capi/Cargo.toml index 7cdf6d499..1b9c92f01 100644 --- a/capi/Cargo.toml +++ b/capi/Cargo.toml @@ -50,6 +50,23 @@ rules-profiling = ["yara-x/rules-profiling"] # This feature is disabled by default. magic-module = ["yara-x/magic-module"] +# Enables internal logging and automatically writes log messages to stderr. +# +# When this feature is enabled, log output can be configured via the `YRX_LOG` +# environment variable set to any of the log levels: `error`, `warn`, `info`, +# `debug`, or `trace`. +# +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info +# YRX_LOG=yara_x=debug +# YRX_LOG=warn,yara_x::compiler=trace +# +# This feature is disabled by default. +stderr-logs = ["yara-x/stderr-logs"] + + [lib] name = "yara_x_capi" crate-type = ["staticlib", "cdylib"] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0bbedc2a0..cf8881798 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -28,8 +28,8 @@ test = false # Enable the "debug" command for developers. debug-cmd = [] -# When this feature is enabled the CLI program prints debug logs if -# the RUST_LOG environment variable is set to any of the debug levels: +# When this feature is enabled the CLI program prints debug logs to stderr if +# the YRX_LOG environment variable is set to any of the log levels: # # error # warn @@ -37,8 +37,13 @@ debug-cmd = [] # debug # trace # -# Example: RUST_LOG=info ./yr scan some_rule.yar some_file -logging = ["dep:log", "dep:env_logger"] +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info ./yr scan some_rule.yar some_file +# YRX_LOG=yara_x=debug ./yr scan some_rule.yar some_file +# YRX_LOG=warn,yara_x::compiler=trace ./yr scan some_rule.yar some_file +stderr-logs = ["yara-x/stderr-logs"] # Enables rules profiling. Notice that profiling has an impact on scan # performance. @@ -58,8 +63,6 @@ figment = { workspace = true, features = ["toml"] } globwalk = { workspace = true } home = { workspace = true } itertools = { workspace = true } -env_logger = { workspace = true, optional = true, features = ["auto-color"] } -log = { workspace = true, optional = true } protobuf = { workspace = true } regex = { workspace = true } serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/cli/src/main.rs b/cli/src/main.rs index 3c7fe115f..ba9d7b0dc 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -35,9 +35,6 @@ fn main() -> anyhow::Result<()> { println!("could not enable ANSI support: {err}") } - #[cfg(feature = "logging")] - env_logger::init(); - // If stdout is not a tty (for example, because it was redirected to a // file) turn off colors. This way you can redirect the output to a file // without ANSI escape codes messing up the file content. diff --git a/cli/src/walk.rs b/cli/src/walk.rs index 674618895..4b7732b52 100644 --- a/cli/src/walk.rs +++ b/cli/src/walk.rs @@ -539,10 +539,10 @@ impl<'a> ParWalker<'a> { } })); - // `multi_progress` will be `None` if the `logging` feature is - // enabled or if either stdout or stderr is not a tty (for example - // when any of them are redirected to a file). - let multi_progress = if cfg!(feature = "logging") { + // `multi_progress` will be `None` if the `stderr-logs` feature + // is enabled or if either stdout or stderr is not a tty (for + // example when any of them are redirected to a file). + let multi_progress = if cfg!(feature = "stderr-logs") { None } else if io::stdout().is_tty() { Some(MultiProgress::new()) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f7949c7a6..ac694cea6 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -34,6 +34,35 @@ rustdoc-args = ["--cfg", "docsrs"] # actually computes the expression. constant-folding = [] +# Enables internal log statements inside YARA-X via the standard `log` crate +# facade. +# +# When this feature is enabled, YARA-X emits internal log messages (using +# `log::info!`,`log::error!`, etc.). However, it does not initialize any +# concrete logger implementation. The host application remains responsible +# for configuring and initializing its own logger (e.g., `env_logger`, +# `tracing-subscriber`, or a custom logger). +logging = ["dep:log", "dep:quanta"] + +# Enables internal logging and automatically logs messages to stderr. +# +# Unlike `logging` (which only emits log events and relies on the host +# application to set up a logger), `stderr-logs` actually writes log messages +# to stderr. This is particularly useful when YARA-X is embedded in non-Rust +# host binaries, allowing log output to be configured via the `YRX_LOG` +# environment variable. +# +# Log output can be configured by setting `YRX_LOG` to any of the log levels: +# `error`, `warn`, `info`, `debug`, or `trace`. +# +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info # Global level set to info +# YRX_LOG=yara_x=debug # Debug level for the yara_x crate +# YRX_LOG=warn,yara_x::compiler=trace # Global warn, trace for yara_x::compiler +stderr-logs = ["dep:env_logger", "logging"] + # Enables the use of exact atoms for speeding up matches. Exact atoms are those # that don't require further verification, the sole presence of the atom # indicates that the pattern containing the atom matches. For instance, in @@ -74,9 +103,6 @@ generate-module-docs = ["protoc"] # plugin. Follow the instructions in: https://lib.rs/crates/protobuf-codegen3 protoc = [] -# Enables debug logs. -logging = ["dep:log", "dep:quanta"] - # When enabled, the serialization of compiled rules include native code for # the platform in which the rules where compiled. This reduces the load time, # as the native code is already included in the serialized rules and doesn't @@ -310,6 +336,7 @@ digest = { workspace = true, optional = true } dsa = { workspace = true, optional = true } ecdsa = { workspace = true, optional = true } encoding_rs = { workspace = true, optional = true } +env_logger = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } memmap2 = { workspace = true } indexmap = { workspace = true, features = ["serde"] } diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 18a9c6e38..34ef44a2f 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -500,6 +500,8 @@ pub struct Compiler<'a> { impl<'a> Compiler<'a> { /// Creates a new YARA compiler. pub fn new() -> Self { + crate::init_logger(); + let mut ident_pool = StringPool::new(); let mut symbol_table = StackedSymbolTable::new(); diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 548226cb0..1617bc61a 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -323,6 +323,8 @@ impl Rules { }); } + crate::init_logger(); + #[cfg(feature = "logging")] let start = Instant::now(); @@ -597,7 +599,7 @@ impl Rules { let rule = self.get(rule_id); - info!( + warn!( "Very short atom in pattern `{}` in rule `{}:{}` (length: {})", self.ident_pool.get(pattern_ident_id).unwrap(), self.ident_pool.get(rule.namespace_ident_id).unwrap(), diff --git a/lib/src/lib.rs b/lib/src/lib.rs index a463d0ee8..b11723575 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -180,3 +180,33 @@ pub unsafe fn finalize() { wasm::free_engine(); } } + +#[cfg(feature = "stderr-logs")] +/// Initializes the `env_logger` backend for logging output to stdout/stderr. +/// +/// This function is called automatically when creating a [`Compiler`] or +/// [`Scanner`] if the `stderr-logs` feature is enabled. It uses +/// `env_logger::try_init()`, which reads the `YRX_LOG` environment variable +/// and safely ignores initialization if a logger was already registered. +pub(crate) fn init_logger() { + static INIT_LOGGER: std::sync::Once = std::sync::Once::new(); + INIT_LOGGER.call_once(|| { + let mut builder = env_logger::Builder::from_env("YRX_LOG"); + + for noisy_module in [ + "cranelift_codegen", + "cranelift_frontend", + "wasmtime", + "wasmtime_internal_cranelift", + "walrus", + ] { + builder.filter_module(noisy_module, log::LevelFilter::Info); + } + + let _ = builder.try_init(); + }); +} + +#[cfg(not(feature = "stderr-logs"))] +#[inline] +pub(crate) fn init_logger() {} diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 57fac1903..076ed00a8 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -744,7 +744,7 @@ impl ScanContext<'_, '_> { let rule = self.compiled_rules.get(rule_id); #[cfg(feature = "logging")] - log::info!( + log::debug!( "Rule match: {}:{} {:?}", self.compiled_rules .ident_pool() @@ -2056,6 +2056,8 @@ impl From for RuntimeObjectHandle { pub fn create_wasm_store_and_ctx<'r>( rules: &'r Rules, ) -> Pin>>> { + crate::init_logger(); + let num_rules = rules.num_rules() as u32; let num_patterns = rules.num_patterns() as u32; From d60a54c8b159f9644e8fe756cf5ec751c0b55d3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:31:17 +0000 Subject: [PATCH 2/7] chore(deps-dev): bump fast-uri from 3.1.2 to 3.1.4 in /ls/editors/code (#717) Bumps [fast-uri](https://github.com/fastify/fast-uri) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.1.2...v3.1.4) --- updated-dependencies: - dependency-name: fast-uri dependency-version: 3.1.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ls/editors/code/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ls/editors/code/package-lock.json b/ls/editors/code/package-lock.json index ead3b61a4..aa884c11b 100644 --- a/ls/editors/code/package-lock.json +++ b/ls/editors/code/package-lock.json @@ -2444,9 +2444,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "dev": true, "funding": [ { From 553b9df31f52d13d2527388f6512c8f7987c5398 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 16:26:24 +0200 Subject: [PATCH 3/7] feat: add warning log when the list of matches for some patterns grows too much. --- lib/src/compiler/rules.rs | 26 +++++---- lib/src/scanner/context.rs | 50 +++++++++++++++--- lib/src/scanner/matches.rs | 105 ++++++++++++++++++++++++++----------- 3 files changed, 132 insertions(+), 49 deletions(-) diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 1617bc61a..863234adc 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -528,13 +528,21 @@ impl Rules { pub(crate) fn get_rule_and_pattern_by_sub_pattern_id( &self, sub_pattern_id: SubPatternId, - ) -> Option<(RuleId, IdentId)> { - let (target_pattern_id, _) = self.get_sub_pattern(sub_pattern_id); - for (rule_id, rule) in self.rules.iter().enumerate() { + ) -> Option<(&RuleInfo, &PatternInfo)> { + let (pattern_id, _) = self.get_sub_pattern(sub_pattern_id); + self.get_rule_and_pattern_by_pattern_id(*pattern_id) + } + + #[cfg(feature = "logging")] + pub(crate) fn get_rule_and_pattern_by_pattern_id( + &self, + pattern_id: PatternId, + ) -> Option<(&RuleInfo, &PatternInfo)> { + for rule in &self.rules { for p in &rule.patterns { - if p.pattern_id == *target_pattern_id { - return Some((rule_id.into(), p.ident_id)); - }; + if p.pattern_id == pattern_id { + return Some((rule, p)); + } } } None @@ -593,15 +601,13 @@ impl Rules { } if x.atom.len() < 2 { - let (rule_id, pattern_ident_id) = self + let (rule, pattern) = self .get_rule_and_pattern_by_sub_pattern_id(x.sub_pattern_id) .unwrap(); - let rule = self.get(rule_id); - warn!( "Very short atom in pattern `{}` in rule `{}:{}` (length: {})", - self.ident_pool.get(pattern_ident_id).unwrap(), + self.ident_pool.get(pattern.ident_id).unwrap(), self.ident_pool.get(rule.namespace_ident_id).unwrap(), self.ident_pool.get(rule.ident_id).unwrap(), x.atom.len() diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 076ed00a8..5074bcc9b 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -34,7 +34,9 @@ use crate::re::hir::ChainedPatternGap; use crate::re::thompson::PikeVM; #[cfg(feature = "rules-profiling")] use crate::scanner::ProfilingData; -use crate::scanner::matches::{Match, PatternMatches, UnconfirmedMatch}; +use crate::scanner::matches::{ + AddResult, Match, PatternMatches, UnconfirmedMatch, +}; use crate::scanner::{DataSnippets, ScanError, ScannedData}; use crate::scanner::{HEARTBEAT_COUNTER, INIT_HEARTBEAT}; use crate::types::{Array, Map, Struct, TypeValue}; @@ -1918,12 +1920,46 @@ fn track_pattern_match( bits.set(pattern_id.into(), true); - let added = - tracker.pattern_matches.add(pattern_id, match_, replace_if_longer); - if !added - || (tracker.fast_scan - && tracker.compiled_rules.is_fast_scan(pattern_id)) - { + // If we are in fast scan mode, and this pattern is suitable to be disabled + // in fast scan mode, disabled it because we already found the first match. + let mut disable_pattern = + tracker.fast_scan && tracker.compiled_rules.is_fast_scan(pattern_id); + + match tracker.pattern_matches.add(pattern_id, match_, replace_if_longer) { + #[cfg(feature = "logging")] + AddResult::Inserted(len) if len % 100_000 == 0 => { + let (rule, pattern) = tracker + .compiled_rules + .get_rule_and_pattern_by_pattern_id(pattern_id) + .unwrap(); + + log::warn!( + "Pattern `{}` in rule `{}:{}` grew to {} matches", + tracker + .compiled_rules + .ident_pool() + .get(pattern.ident_id) + .unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.namespace_ident_id) + .unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.ident_id) + .unwrap(), + len + ); + } + AddResult::MaxMatchesReached => { + disable_pattern = true; + } + _ => {} + } + + if disable_pattern { tracker.disabled_patterns.insert(pattern_id); } } diff --git a/lib/src/scanner/matches.rs b/lib/src/scanner/matches.rs index 53c5a4e74..5f625ea99 100644 --- a/lib/src/scanner/matches.rs +++ b/lib/src/scanner/matches.rs @@ -69,16 +69,19 @@ impl MatchList { /// is false, the existing match will remain untouched and the new one will /// be ignored. /// + /// Returns `true` if a new match was added to the list, or `false` if an + /// existing match was updated or ignored. + /// /// This operation is O(1) in the most common case, which is inserting /// the new match at an offset that is higher than those from previous /// matches. - pub fn add(&mut self, new_match: Match, replace_if_longer: bool) { + pub fn add(&mut self, new_match: Match, replace_if_longer: bool) -> bool { match self.matches.last_mut() { // The new match starts at some offset greater than the offset of // the last match, we can simply push the new match at the end. Some(last) if new_match.range.start > last.range.start => { self.matches.push(new_match); - return; + true } // The new match starts has the same offset as the last match. We // only need to update its end position when replace_if_longer is @@ -87,34 +90,38 @@ impl MatchList { if replace_if_longer { last.range.end = new_match.range.end; } - return; + false } // No matches so far, the match is first one. None => { self.matches.push(new_match); - return; + true } - _ => {} - } - // If not at the end, use binary search to find the insertion point. - match self - .matches - .binary_search_by_key(&new_match.range.start, |m| m.range.start) - { - // Found, and replace_if_longer is true. - Ok(index) if replace_if_longer => { - let existing_match = &mut self.matches[index]; - if existing_match.range.end < new_match.range.end { - existing_match.range.end = new_match.range.end; + _ => { + // If not at the end, use binary search to find the insertion point. + match self + .matches + .binary_search_by_key(&new_match.range.start, |m| { + m.range.start + }) { + // Found, and replace_if_longer is true. + Ok(index) if replace_if_longer => { + let existing_match = &mut self.matches[index]; + if existing_match.range.end < new_match.range.end { + existing_match.range.end = new_match.range.end; + } + false + } + // Not found, insert at the corresponding position. + Err(index) => { + self.matches.insert(index, new_match); + true + } + // Found, but replace_if_longer is false, do nothing and keep the + // existing match. + _ => false, } } - // Not found, insert at the corresponding position. - Err(index) => { - self.matches.insert(index, new_match); - } - // Found, but replace_if_longer is false, do nothing and keep the - // existing match. - _ => {} } } @@ -213,6 +220,16 @@ pub struct UnconfirmedMatch { pub chain_length: usize, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AddResult { + /// A new match was inserted into the list, growing it to the given length. + Inserted(usize), + /// An existing match was updated or ignored (list length did not increase). + Updated, + /// Pattern has reached `max_matches_per_pattern` limit and the match was rejected. + MaxMatchesReached, +} + /// A hash map that tracks matches for each pattern. /// /// Each key in this map is a [`PatternId`], and its associated value is a @@ -288,33 +305,37 @@ impl PatternMatches { /// replaced. If the argument is `false` the new match will be ignored and /// the existing one will remain. /// - /// This function returns `true` if the new match was added, or `false` - /// if the pattern already reached the maximum number of matches and - /// therefore the new match was not added. + /// Returns [`AddResult`] indicating whether a new match was inserted, an + /// existing match was updated, or the maximum match limit was reached. pub fn add( &mut self, pattern_id: PatternId, m: Match, replace_if_longer: bool, - ) -> bool { + ) -> AddResult { match self.matches.entry(pattern_id) { Entry::Occupied(mut entry) => { let matches = entry.get_mut(); if matches.len() < self.max_matches_per_pattern { self.capacity -= matches.capacity(); - matches.add(m, replace_if_longer); + let inserted = matches.add(m, replace_if_longer); self.capacity += matches.capacity(); - true + if inserted { + AddResult::Inserted(matches.len()) + } else { + AddResult::Updated + } } else { - false + AddResult::MaxMatchesReached } } Entry::Vacant(entry) => { let mut matches = MatchList::with_capacity(8); self.capacity += matches.capacity(); matches.add(m, replace_if_longer); + let len = matches.len(); entry.insert(matches); - true + AddResult::Inserted(len) } } } @@ -328,7 +349,7 @@ impl PatternMatches { #[cfg(test)] mod test { - use crate::scanner::matches::{Match, MatchList}; + use crate::scanner::matches::{Match, MatchList, PatternMatches}; use std::ops::Range; #[test] @@ -347,4 +368,24 @@ mod test { vec![(1..15), (2..10), (3..10), (4..10), (5..10)] ) } + + #[test] + fn match_list_growth() { + let mut ml = MatchList::with_capacity(10_000); + for i in 0..10_000 { + ml.add(Match::new(i..i + 1), false); + } + assert_eq!(ml.len(), 10_000); + } + + #[test] + fn pattern_matches_growth() { + use crate::compiler::PatternId; + let mut pm = PatternMatches::new(); + let pid = PatternId::from(0); + for i in 0..10_000 { + pm.add(pid, Match::new(i..i + 1), false); + } + assert_eq!(pm.get(pid).unwrap().len(), 10_000); + } } From 1e3338a69ca9a17c823ae744739c1a784c7dfaec Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 16:36:19 +0200 Subject: [PATCH 4/7] feat: add warning log for excessive unconfirmed matches. --- lib/src/scanner/context.rs | 66 ++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 5074bcc9b..48584ded5 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -1440,7 +1440,7 @@ fn verify_chain_of_matches( match &tracker.compiled_rules.get_sub_pattern(id).1 { SubPattern::LiteralChainHead { flags, .. } | SubPattern::RegexpChainHead { flags, .. } => { - track_pattern_match( + track_match( tracker, wasm_state, pattern_id, @@ -1851,12 +1851,10 @@ fn handle_sub_pattern_match( | SubPattern::Base64Wide { .. } | SubPattern::CustomBase64 { .. } | SubPattern::CustomBase64Wide { .. } => { - track_pattern_match( - tracker, wasm_state, pattern_id, match_, false, - ); + track_match(tracker, wasm_state, pattern_id, match_, false); } SubPattern::Regexp { flags, .. } => { - track_pattern_match( + track_match( tracker, wasm_state, pattern_id, @@ -1865,11 +1863,13 @@ fn handle_sub_pattern_match( ); } SubPattern::LiteralChainHead { .. } - | SubPattern::RegexpChainHead { .. } => tracker - .unconfirmed_matches - .entry(sub_pattern_id) - .or_default() - .push(UnconfirmedMatch { range: match_.range, chain_length: 0 }), + | SubPattern::RegexpChainHead { .. } => { + track_unconfirmed_match( + tracker, + sub_pattern_id, + UnconfirmedMatch { range: match_.range, chain_length: 0 }, + ); + } SubPattern::LiteralChainTail { chained_to, gap, flags, .. } | SubPattern::RegexpChainTail { chained_to, gap, flags, .. } => { if within_valid_distance( @@ -1887,21 +1887,53 @@ fn handle_sub_pattern_match( match_, ); } else { - tracker - .unconfirmed_matches - .entry(sub_pattern_id) - .or_default() - .push(UnconfirmedMatch { + track_unconfirmed_match( + tracker, + sub_pattern_id, + UnconfirmedMatch { range: match_.range, chain_length: 0, - }); + }, + ); } } } } } -fn track_pattern_match( +#[inline] +fn track_unconfirmed_match( + tracker: &mut MatchTracker, + sub_pattern_id: SubPatternId, + unconfirmed_match: UnconfirmedMatch, +) { + let unconfirmed_matches = + tracker.unconfirmed_matches.entry(sub_pattern_id).or_default(); + + unconfirmed_matches.push(unconfirmed_match); + + #[cfg(feature = "logging")] + if unconfirmed_matches.len() % 100_000 == 0 { + let (rule, pattern) = tracker + .compiled_rules + .get_rule_and_pattern_by_sub_pattern_id(sub_pattern_id) + .unwrap(); + + log::warn!( + "Pattern `{}` in rule `{}:{}` grew to {} unconfirmed matches", + tracker.compiled_rules.ident_pool().get(pattern.ident_id).unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.namespace_ident_id) + .unwrap(), + tracker.compiled_rules.ident_pool().get(rule.ident_id).unwrap(), + unconfirmed_matches.len() + ); + } +} + +fn track_match( tracker: &mut MatchTracker, wasm_state: &mut WasmState, pattern_id: PatternId, From a33fa8e45afb37e1a53a34db6a455929453879a5 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:15:48 +0200 Subject: [PATCH 5/7] feat: add `yrx_scanner_max_matches_per_pattern` to the C API and `Scanner.MaxMatchedPerPattern` to the Go API. --- capi/include/yara_x.h | 7 +++++++ capi/src/scanner.rs | 32 +++++++++++++++++++++++++++++ capi/src/tests.rs | 41 +++++++++++++++++++++++++++++++++++++- go/scanner.go | 6 ++++++ go/scanner_test.go | 16 +++++++++++++++ py/src/lib.rs | 3 ++- site/content/docs/api/c.md | 15 ++++++++++++++ 7 files changed, 118 insertions(+), 2 deletions(-) diff --git a/capi/include/yara_x.h b/capi/include/yara_x.h index ee6f0eb5f..ace19eceb 100644 --- a/capi/include/yara_x.h +++ b/capi/include/yara_x.h @@ -716,6 +716,13 @@ enum YRX_RESULT yrx_scanner_set_timeout(struct YRX_SCANNER *scanner, enum YRX_RESULT yrx_scanner_fast_scan(struct YRX_SCANNER *scanner, bool yes); +// Sets the maximum number of matches per pattern. +// +// When a pattern reaches the maximum number of matches it won't produce more +// matches. +enum YRX_RESULT yrx_scanner_max_matches_per_pattern(struct YRX_SCANNER *scanner, + size_t n); + // Scans a data buffer. // // `data` can be null as long as `len` is 0. In such cases its handled as diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs index 71103c0b7..af1073629 100644 --- a/capi/src/scanner.rs +++ b/capi/src/scanner.rs @@ -47,6 +47,19 @@ impl<'r> InnerScanner<'r> { self } + fn max_matches_per_pattern(&mut self, n: usize) -> &mut Self { + match self { + InnerScanner::SingleBlock(s) => { + s.max_matches_per_pattern(n); + } + InnerScanner::MultiBlock(s) => { + s.max_matches_per_pattern(n); + } + InnerScanner::None => unreachable!(), + } + self + } + fn make_multi_block(&mut self) -> &mut yara_x::blocks::Scanner<'r> { // Already a multi-block scanner, nothing else to do. if let Self::MultiBlock(s) = self { @@ -193,6 +206,25 @@ pub unsafe extern "C" fn yrx_scanner_fast_scan( YRX_RESULT::YRX_SUCCESS } +/// Sets the maximum number of matches per pattern. +/// +/// When a pattern reaches the maximum number of matches it won't produce more +/// matches. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn yrx_scanner_max_matches_per_pattern( + scanner: *mut YRX_SCANNER, + n: usize, +) -> YRX_RESULT { + let scanner = match scanner.as_mut() { + Some(s) => s, + None => return YRX_RESULT::YRX_INVALID_ARGUMENT, + }; + + scanner.inner.max_matches_per_pattern(n); + + YRX_RESULT::YRX_SUCCESS +} + /// Scans a data buffer. /// /// `data` can be null as long as `len` is 0. In such cases its handled as diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 5163a7ea6..7d9b6f3d8 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -18,7 +18,8 @@ use crate::{ yrx_rules_destroy, yrx_rules_iter, yrx_rules_iter_imports, yrx_rules_serialize, yrx_scanner_clear_profiling_data, yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_fast_scan, yrx_scanner_finish, - yrx_scanner_iter_slowest_rules, yrx_scanner_on_console_log, + yrx_scanner_iter_slowest_rules, yrx_scanner_max_matches_per_pattern, + yrx_scanner_on_console_log, yrx_scanner_on_matching_rule, yrx_scanner_scan, yrx_scanner_scan_block, yrx_scanner_scan_file, yrx_scanner_set_global_bool, yrx_scanner_set_global_float, yrx_scanner_set_global_int, @@ -571,6 +572,40 @@ fn capi_fast_scan() { } } +#[test] +fn capi_max_matches_per_pattern() { + unsafe { + let mut compiler = std::ptr::null_mut(); + yrx_compiler_create(0, &mut compiler); + + let src = c"rule test { strings: $a = \"foo\" condition: #a == 1 }"; + yrx_compiler_add_source(compiler, src.as_ptr()); + + let rules = yrx_compiler_build(compiler); + yrx_compiler_destroy(compiler); + + let mut scanner = std::ptr::null_mut(); + yrx_scanner_create(rules, &mut scanner); + + yrx_scanner_max_matches_per_pattern(scanner, 1); + + let mut matches = 0; + yrx_scanner_on_matching_rule( + scanner, + on_rule_match_increase_counter, + &mut matches as *mut i32 as *mut c_void, + ); + + let data = b"foofoofoo"; + yrx_scanner_scan(scanner, data.as_ptr(), data.len()); + + assert_eq!(matches, 1); + + yrx_scanner_destroy(scanner); + yrx_rules_destroy(rules); + } +} + #[test] fn capi_null_args() { unsafe { @@ -775,6 +810,10 @@ fn capi_null_args() { yrx_scanner_fast_scan(std::ptr::null_mut(), true), YRX_RESULT::YRX_INVALID_ARGUMENT ); + assert_eq!( + yrx_scanner_max_matches_per_pattern(std::ptr::null_mut(), 10), + YRX_RESULT::YRX_INVALID_ARGUMENT + ); assert_eq!( yrx_scanner_scan(std::ptr::null_mut(), std::ptr::null(), 0), YRX_RESULT::YRX_INVALID_ARGUMENT diff --git a/go/scanner.go b/go/scanner.go index 67c08ad8f..ce3a5a76d 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -128,6 +128,12 @@ func (s *Scanner) FastScan(yes bool) { runtime.KeepAlive(s) } +// MaxMatchesPerPattern sets the maximum number of matches per pattern. +func (s *Scanner) MaxMatchesPerPattern(n int) { + C.yrx_scanner_max_matches_per_pattern(s.cScanner, C.size_t(n)) + runtime.KeepAlive(s) +} + var ErrTimeout = errors.New("timeout") // SetGlobal sets the value of a global variable. diff --git a/go/scanner_test.go b/go/scanner_test.go index 49dd363cc..648b6b7b9 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -178,3 +178,19 @@ func TestScannerFastScan(t *testing.T) { assert.Len(t, matchingRules[0].Patterns(), 1) assert.Len(t, matchingRules[0].Patterns()[0].Matches(), 1) } + +func TestScannerMaxMatchesPerPattern(t *testing.T) { + r, _ := Compile(` + rule t { + strings: + $a = "foo" + condition: + #a == 1 + }`) + s := NewScanner(r) + s.MaxMatchesPerPattern(1) + scanResults, _ := s.Scan([]byte("foofoofoo")) + matchingRules := scanResults.MatchingRules() + + assert.Len(t, matchingRules, 1) +} diff --git a/py/src/lib.rs b/py/src/lib.rs index edf57ac29..2b04be5d0 100644 --- a/py/src/lib.rs +++ b/py/src/lib.rs @@ -1086,7 +1086,8 @@ impl Scanner { /// Sets the maximum number of matches per pattern. /// - /// When some pattern reaches the specified number of `matches` it won't produce more matches. + /// When some pattern reaches the specified number of `matches` it won't + /// produce more matches. fn max_matches_per_pattern(&mut self, matches: usize) { self.inner.max_matches_per_pattern(matches); } diff --git a/site/content/docs/api/c.md b/site/content/docs/api/c.md index 3d6c926d6..928e26c6e 100644 --- a/site/content/docs/api/c.md +++ b/site/content/docs/api/c.md @@ -764,6 +764,21 @@ pattern in the file, only the first one. ------ +### yrx_scanner_max_matches_per_pattern + +```c +enum YRX_RESULT yrx_scanner_max_matches_per_pattern( + struct YRX_SCANNER *scanner, + size_t n); +``` + +Sets the maximum number of matches per pattern. + +When a pattern reaches the maximum number of matches it won't produce more +matches. + +------ + ### yrx_scanner_set_global_xxxx ```c From 99a72a57d338242d48095efeb4e48e88febfaafb Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:16:47 +0200 Subject: [PATCH 6/7] chore: update `field_docs.rs`. --- lib/src/modules/field_docs.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/src/modules/field_docs.rs b/lib/src/modules/field_docs.rs index 7edc6d01a..d994d10f4 100644 --- a/lib/src/modules/field_docs.rs +++ b/lib/src/modules/field_docs.rs @@ -343,6 +343,8 @@ pub const FIELD_DOCS: &[(&str, u64, &str)] = &[ ("macho.Symtab", 4, "Size in bytes of the string table."), ("macho.Symtab", 5, "Individual entries stored in the table."), ("macho.Symtab", 6, "Descriptive nlist entries for symbols."), + ("msi.Msi", 1, "True if the file is an MSI file and contains a digital signature."), + ("msi.Msi", 2, "Digital signatures present in the MSI file."), ("olecf.Olecf", 1, "True if file is an OLE CF file."), ("olecf.Olecf", 2, "Streams contained in the OLE CF file."), ("pe.Certificate", 1, "Issuer of this individual certificate."), From cb7fb566b5e3a65cf8574e665e3b2ddc57084e7e Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:36:23 +0200 Subject: [PATCH 7/7] style: run `cargo fmt`. --- capi/src/tests.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 7d9b6f3d8..149a68e69 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -19,13 +19,12 @@ use crate::{ yrx_rules_serialize, yrx_scanner_clear_profiling_data, yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_fast_scan, yrx_scanner_finish, yrx_scanner_iter_slowest_rules, yrx_scanner_max_matches_per_pattern, - yrx_scanner_on_console_log, - yrx_scanner_on_matching_rule, yrx_scanner_scan, yrx_scanner_scan_block, - yrx_scanner_scan_file, yrx_scanner_set_global_bool, - yrx_scanner_set_global_float, yrx_scanner_set_global_int, - yrx_scanner_set_global_json, yrx_scanner_set_global_str, - yrx_scanner_set_module_data, yrx_scanner_set_module_output, - yrx_scanner_set_timeout, + yrx_scanner_on_console_log, yrx_scanner_on_matching_rule, + yrx_scanner_scan, yrx_scanner_scan_block, yrx_scanner_scan_file, + yrx_scanner_set_global_bool, yrx_scanner_set_global_float, + yrx_scanner_set_global_int, yrx_scanner_set_global_json, + yrx_scanner_set_global_str, yrx_scanner_set_module_data, + yrx_scanner_set_module_output, yrx_scanner_set_timeout, }; use std::ffi::{CStr, c_char, c_void};