From cb22022f5579ad5252fc5aa923c26d3105844a10 Mon Sep 17 00:00:00 2001 From: rabbitstack Date: Thu, 16 Jul 2026 22:23:08 +0200 Subject: [PATCH 1/4] build(deps): Add golang-lru and xxh3 deps --- go.mod | 6 +++++- go.sum | 29 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 839eccae8..8510793f0 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/enescakir/emoji v1.0.0 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/hashicorp/go-version v1.2.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hillu/go-yara/v4 v4.2.4 github.com/jedib0t/go-pretty/v6 v6.2.1 github.com/lithammer/fuzzysearch v1.1.2 @@ -33,6 +34,7 @@ require ( github.com/valyala/gozstd v1.11.0 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yuin/goldmark v1.5.2 + github.com/zeebo/xxh3 v1.1.0 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 golang.org/x/arch v0.6.0 golang.org/x/sys v0.31.0 @@ -41,11 +43,12 @@ require ( gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 - www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc ) require ( github.com/BurntSushi/toml v0.4.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d // indirect @@ -54,6 +57,7 @@ require ( golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect honnef.co/go/tools v0.3.2 // indirect ) diff --git a/go.sum b/go.sum index c11b25fdb..dc48d1a7d 100644 --- a/go.sum +++ b/go.sum @@ -11,12 +11,6 @@ github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFP github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= -github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= -github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= -github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= -github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= @@ -40,6 +34,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -91,6 +86,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hillu/go-yara/v4 v4.2.4 h1:r3KB1XV+h6q+N8bvK6/gLpxAVcd6baYzmOSYHzNo9QQ= @@ -112,14 +109,18 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lithammer/fuzzysearch v1.1.2 h1:ePUtm14xKxbpCxozcFbIDRtvANxnVnE+RKpJUqkr2gA= github.com/lithammer/fuzzysearch v1.1.2/go.mod h1:v6tYW/9kpfV6LNcweXdSjQsfCku/1M/oObmSox1fzP8= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= @@ -151,6 +152,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -175,16 +177,13 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/saferwall/pe v1.5.6 h1:DrRLnoQFxHWJ5lJUmrH7X2L0xeUu6SUS95Dc61eW2Yc= github.com/saferwall/pe v1.5.6/go.mod h1:mJx+PuptmNpoPFBNhWs/uDMFL/kTHVZIkg0d4OUJFbQ= -github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= -github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d h1:RQqyEogx5J6wPdoxqL132b100j8KjcVHO1c0KLRoIhc= github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d/go.mod h1:PegD7EVqlN88z7TpCqH92hHP+GBpfomGCCnw1PFtNOA= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -251,6 +250,10 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU= github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= @@ -303,8 +306,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -385,5 +386,3 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= -www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc h1:eeL+RUEGr6/lYL8hJEbvugrF88I6W4pBaVtFa1falj4= -www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= From 780cf26ac22ad1b4655759e760741e1ffccb7587 Mon Sep 17 00:00:00 2001 From: rabbitstack Date: Thu, 16 Jul 2026 22:25:57 +0200 Subject: [PATCH 2/4] perf(filter): Implement case sensitive/insensitive wildcard matching --- pkg/filter/ql/ast.go | 25 ++------ pkg/filter/ql/functions/count.go | 8 +-- pkg/fs/file.go | 2 +- pkg/ps/types/types_windows.go | 2 +- pkg/util/wildcard/wildcard.go | 97 ++++++++++++++++++++++++------ pkg/util/wildcard/wildcard_test.go | 29 +++++---- pkg/yara/config/config.go | 10 +-- 7 files changed, 112 insertions(+), 61 deletions(-) diff --git a/pkg/filter/ql/ast.go b/pkg/filter/ql/ast.go index 124dfd542..aec705db2 100644 --- a/pkg/filter/ql/ast.go +++ b/pkg/filter/ql/ast.go @@ -30,18 +30,6 @@ import ( "github.com/rabbitstack/fibratus/pkg/util/wildcard" ) -// loweredWildcards contains a mapping between original and lower-case wildcard patterns -var loweredWildcards = map[string]string{} - -func toLowerWildcard(p string) string { - if s, ok := loweredWildcards[p]; ok { - return s - } - s := strings.ToLower(p) - loweredWildcards[p] = s - return s -} - // Eval evaluates expr against a map that contains the field values. func Eval(expr Expr, m map[string]interface{}, useFuncValuer bool) bool { var eval ValuerEval @@ -961,10 +949,10 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { case Matches: switch rhs := rhs.(type) { case string: - return wildcard.Match(rhs, lhs) + return wildcard.Match(rhs, lhs, true) case []string: for _, pat := range rhs { - if wildcard.Match(pat, lhs) { + if wildcard.Match(pat, lhs, true) { return true } } @@ -973,13 +961,12 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { return false } case IMatches: - v := strings.ToLower(lhs) switch rhs := rhs.(type) { case string: - return wildcard.Match(toLowerWildcard(rhs), v) + return wildcard.Match(rhs, lhs, false) case []string: for _, pat := range rhs { - if wildcard.Match(toLowerWildcard(pat), v) { + if wildcard.Match(pat, lhs, false) { return true } } @@ -1231,7 +1218,7 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { } for _, pat := range rhs { for _, val := range lhs { - if wildcard.Match(pat, val) { + if wildcard.Match(pat, val, true) { return true } } @@ -1244,7 +1231,7 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { } for _, pat := range rhs { for _, val := range lhs { - if wildcard.Match(toLowerWildcard(pat), strings.ToLower(val)) { + if wildcard.Match(pat, val, false) { return true } } diff --git a/pkg/filter/ql/functions/count.go b/pkg/filter/ql/functions/count.go index 1ed051659..e1cb8842a 100644 --- a/pkg/filter/ql/functions/count.go +++ b/pkg/filter/ql/functions/count.go @@ -50,11 +50,11 @@ func (f Count) Call(args []interface{}) (any, bool) { for _, ss := range substrings { switch caseInsensitive { case true: - if wildcard.Match(strings.ToLower(pattern), strings.ToLower(ss)) { + if wildcard.Match(pattern, ss, false) { count++ } case false: - if wildcard.Match(pattern, ss) { + if wildcard.Match(pattern, ss, true) { count++ } } @@ -63,11 +63,11 @@ func (f Count) Call(args []interface{}) (any, bool) { for _, i := range s { switch caseInsensitive { case true: - if wildcard.Match(strings.ToLower(pattern), strings.ToLower(i)) { + if wildcard.Match(pattern, i, false) { count++ } case false: - if wildcard.Match(pattern, i) { + if wildcard.Match(pattern, i, true) { count++ } } diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 003e6ad24..202d32bcc 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -84,7 +84,7 @@ var parserOpts = []pe.Option{ // from the Portable Executable headers. func GetFileInfo(path string) (*FileInfo, error) { for _, pat := range skippedPatterns { - if wildcard.Match(pat, path) { + if wildcard.Match(pat, path, false) { return nil, ErrSkippedFile(path) } } diff --git a/pkg/ps/types/types_windows.go b/pkg/ps/types/types_windows.go index f602ed5c7..b9a167066 100644 --- a/pkg/ps/types/types_windows.go +++ b/pkg/ps/types/types_windows.go @@ -321,7 +321,7 @@ func (ps *PS) IsAppinfoSvc() bool { // IsSvchost returns true if this is the Service Host process. func (ps *PS) IsSvchost() bool { - return wildcard.Match(`?:\windows\system32\svchost.exe`, strings.ToLower(ps.Exe)) + return wildcard.Match(`?:\windows\system32\svchost.exe`, ps.Exe, false) } // Thread stores metadata about a thread that's executing in process's address space. diff --git a/pkg/util/wildcard/wildcard.go b/pkg/util/wildcard/wildcard.go index b0f7d8f70..23f2fd664 100644 --- a/pkg/util/wildcard/wildcard.go +++ b/pkg/util/wildcard/wildcard.go @@ -12,21 +12,30 @@ package wildcard -import "unicode/utf8" +import ( + "unicode" + "unicode/utf8" +) // Match performs ASCII-first, iterative wildcard matching with UTF-8 fallback. -// It supports '*' and '?' wildcards in the pattern string. -func Match(pattern, str string) bool { - slen := len(str) - plen := len(pattern) +// It supports '*' and '?' wildcards in the pattern string. If caseSensitive +// is false, comparisons are performed case-insensitively with zero extra +// allocations. +func Match(pattern, str string, caseSensitive bool) bool { + if caseSensitive { + return matchCaseSensitive(pattern, str) + } + return matchCaseInsensitive(pattern, str) +} +func matchCaseSensitive(pattern, str string) bool { + slen, plen := len(str), len(pattern) var p, s int wildcardIdx, matchIdx := -1, 0 for s < slen { if p < plen { pb := pattern[p] - switch pb { case '?': // match exactly one character @@ -40,18 +49,14 @@ func Match(pattern, str string) bool { s += ssize } continue - case '*': - // record wildcard position - wildcardIdx = p - matchIdx = s + wildcardIdx, matchIdx = p, s p++ continue - default: // literal match if pb < utf8.RuneSelf && str[s] < utf8.RuneSelf { - if pb == str[s] { + if pb == str[s] { // direct compare, no function call, inlines trivially p++ s++ continue @@ -67,23 +72,79 @@ func Match(pattern, str string) bool { } } } - - // backtrack if there was a previous '*' if wildcardIdx != -1 { p = wildcardIdx + 1 matchIdx++ s = matchIdx continue } - - // previous '*', and mismatch return false } - - // Skip remaining stars in pattern for p < plen && pattern[p] == '*' { p++ } + return p == plen +} + +func matchCaseInsensitive(pattern, str string) bool { + slen, plen := len(str), len(pattern) + var p, s int + wildcardIdx, matchIdx := -1, 0 + for s < slen { + if p < plen { + pb := pattern[p] + switch pb { + case '?': + if str[s] < utf8.RuneSelf && pb < utf8.RuneSelf { + p++ + s++ + } else { + _, psize := utf8.DecodeRuneInString(pattern[p:]) + _, ssize := utf8.DecodeRuneInString(str[s:]) + p += psize + s += ssize + } + continue + case '*': + wildcardIdx, matchIdx = p, s + p++ + continue + default: + if pb < utf8.RuneSelf && str[s] < utf8.RuneSelf { + if pb == str[s] || foldASCII(pb) == foldASCII(str[s]) { // pb==str[s] short-circuits the common exact-match case + p++ + s++ + continue + } + } else { + pr, psize := utf8.DecodeRuneInString(pattern[p:]) + sr, ssize := utf8.DecodeRuneInString(str[s:]) + if pr == sr || unicode.ToLower(pr) == unicode.ToLower(sr) { + p += psize + s += ssize + continue + } + } + } + } + if wildcardIdx != -1 { + p = wildcardIdx + 1 + matchIdx++ + s = matchIdx + continue + } + return false + } + for p < plen && pattern[p] == '*' { + p++ + } return p == plen } + +func foldASCII(b byte) byte { + if b >= 'A' && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} diff --git a/pkg/util/wildcard/wildcard_test.go b/pkg/util/wildcard/wildcard_test.go index 19eb90f07..a993a42c0 100644 --- a/pkg/util/wildcard/wildcard_test.go +++ b/pkg/util/wildcard/wildcard_test.go @@ -20,24 +20,27 @@ import ( func TestMatch(t *testing.T) { var tests = []struct { - p string - s string - match bool + p string + s string + caseSensitive bool + match bool }{ - {"C:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true}, - {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true}, - {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\cmd.exe", false}, - {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\ActionList.xml", true}, - {"C:\\ProgramData\\*.dll", "C:\\ProgramData\\Directory\\OneMoreDirectory\\mal.dll", true}, - {"HKEY_USERS\\*\\Environment\\windir", "HKEY_USERS\\S-1-5-21-2271034452-2606270099-984871569-1001\\Environment\\windir", true}, - {"C:\\Windows\\SoftwareDistribution\\*", "C:\\Windows\\SoftwareDistribution\\SLS\\7971F918-A847-4430-9279-4A52D1EFE18D\\sls.rar", true}, - {"HKEY_USERS\\S-1-5-21-*_CLASSES\\MS-SETTINGS\\CURVER", "HKEY_USERS\\S-1-5-21-2271034452-1207270099-244871569-1021_CLASSES\\MS-SETTINGS\\CURVER", true}, - {"ntdll.dll|KernelBase.dll|advapi32.dll|*", "ntdll.dll|KernelBase.dll|advapi32.dll|pe386.dll|com.dll|clr.dll|mmc.exe", true}, + {"C:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, true}, + {"C:\\*\\LSASS?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, false}, + {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, true}, + {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\cmd.exe", true, false}, + {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\ActionList.xml", true, true}, + {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\actionList.xml", true, false}, + {"C:\\ProgramData\\*.dll", "C:\\ProgramData\\Directory\\OneMoreDirectory\\mal.dll", true, true}, + {"HKEY_USERS\\*\\Environment\\windir", "HKEY_USERS\\S-1-5-21-2271034452-2606270099-984871569-1001\\Environment\\windir", true, true}, + {"C:\\Windows\\SoftwareDistribution\\*", "C:\\Windows\\SoftwareDistribution\\SLS\\7971F918-A847-4430-9279-4A52D1EFE18D\\sls.rar", true, true}, + {"HKEY_USERS\\S-1-5-21-*_CLASSES\\MS-SETTINGS\\CURVER", "HKEY_USERS\\S-1-5-21-2271034452-1207270099-244871569-1021_CLASSES\\MS-SETTINGS\\CURVER", true, true}, + {"ntdll.dll|KernelBase.dll|advapi32.dll|*", "ntdll.dll|KernelBase.dll|advapi32.dll|pe386.dll|com.dll|clr.dll|mmc.exe", true, true}, } for _, tt := range tests { t.Run(tt.p, func(t *testing.T) { - assert.Equal(t, tt.match, Match(tt.p, tt.s)) + assert.Equal(t, tt.match, Match(tt.p, tt.s, tt.caseSensitive)) }) } } diff --git a/pkg/yara/config/config.go b/pkg/yara/config/config.go index 084adfdc0..c3385b116 100644 --- a/pkg/yara/config/config.go +++ b/pkg/yara/config/config.go @@ -21,6 +21,9 @@ package config import ( "bytes" "fmt" + "text/template" + "time" + "github.com/mitchellh/mapstructure" "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" @@ -28,9 +31,6 @@ import ( ytypes "github.com/rabbitstack/fibratus/pkg/yara/types" "github.com/spf13/pflag" "github.com/spf13/viper" - "strings" - "text/template" - "time" ) const ( @@ -143,7 +143,7 @@ func AddFlags(flags *pflag.FlagSet) { // Wildcard matching is possible. func (c Config) ShouldSkipProcess(proc string) bool { for _, p := range c.ExcludedProcesses { - if wildcard.Match(strings.ToLower(p), strings.ToLower(proc)) { + if wildcard.Match(p, proc, false) { return true } } @@ -153,7 +153,7 @@ func (c Config) ShouldSkipProcess(proc string) bool { // ShouldSkipFile determines whether the specified full file path is rejected by the scanner. func (c Config) ShouldSkipFile(file string) bool { for _, f := range c.ExcludedFiles { - if wildcard.Match(strings.ToLower(f), strings.ToLower(file)) { + if wildcard.Match(f, file, false) { return true } } From 8b7fa0b21d7a888256c1fd718943e23c880f340d Mon Sep 17 00:00:00 2001 From: rabbitstack Date: Fri, 17 Jul 2026 17:53:01 +0200 Subject: [PATCH 3/4] perf(approver): Pool event record, buffers and callstack The approver path is allocation heavy, because we have to clone the CreateFile event so it can outlive the ETW callback. To alleviate the pressure on the heap allocator, we agressively pool the event records, their data buffers, and callstack return addresses. --- pkg/sys/etw/record_pool.go | 85 ++++++++++++++++++++++++++++++++++++++ pkg/sys/etw/types.go | 56 +++++++++++++++++-------- pkg/util/va/address.go | 29 ++++++++++++- 3 files changed, 151 insertions(+), 19 deletions(-) create mode 100644 pkg/sys/etw/record_pool.go diff --git a/pkg/sys/etw/record_pool.go b/pkg/sys/etw/record_pool.go new file mode 100644 index 000000000..c56156af3 --- /dev/null +++ b/pkg/sys/etw/record_pool.go @@ -0,0 +1,85 @@ +//go:build windows +// +build windows + +/* + * Copyright 2019-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etw + +import ( + "sync" + "unsafe" +) + +const bufSizeHint = 512 + +var bufPool = sync.Pool{ + New: func() any { + b := make([]byte, 0, bufSizeHint) + return &b + }, +} + +var eventRecordPool = sync.Pool{ + New: func() any { return &EventRecord{} }, +} + +var fileExtDataPool = sync.Pool{ + New: func() any { return &FileExtendedDataItems{} }, +} + +func getBuf(n int) []byte { + bp := bufPool.Get().(*[]byte) + b := *bp + if cap(b) < n { + return make([]byte, n) // rare oversized event, pay the allocation once, don't repool it later + } + return b[:n] +} + +func releaseBuf(b []byte) { + if cap(b) > bufSizeHint*8 { + return // drop outlier-sized buffers instead of pooling them permanently + } + b = b[:0] + bufPool.Put(&b) +} + +func getEventRecord() *EventRecord { + return eventRecordPool.Get().(*EventRecord) +} + +func getFileExtendedDataItems() *FileExtendedDataItems { + return fileExtDataPool.Get().(*FileExtendedDataItems) +} + +func (f *FileExtendedDataItems) ReleasePool() { + f.callstack = nil // release reference, don't let pool retain a stale slice + fileExtDataPool.Put(f) +} + +// ReleasePool returns both the record and its backing buffer to their pools. +// Call only once the record is fully done being read by all downstream consumers. +func (r *EventRecord) ReleasePool() { + if r.Buffer != 0 && r.BufferLen > 0 { + buf := unsafe.Slice((*byte)(unsafe.Pointer(r.Buffer)), r.BufferLen) + releaseBuf(buf) + } + *r = EventRecord{} // zero every field before returning to pool + eventRecordPool.Put(r) +} diff --git a/pkg/sys/etw/types.go b/pkg/sys/etw/types.go index 2a217c920..ccb2d5d25 100644 --- a/pkg/sys/etw/types.go +++ b/pkg/sys/etw/types.go @@ -584,17 +584,13 @@ type FileExtendedDataItems struct { status uint32 disposition uint32 callstack []va.Address - items []EventHeaderExtendedDataItem + items [3]EventHeaderExtendedDataItem } // AppendEventHeaderFileExtendedDataItems appends custom file extendeed data items to the event record. func AppendEventHeaderFileExtendedDataItems(r *EventRecord, disposition uint64, status uint32, callstack []va.Address) *FileExtendedDataItems { - f := &FileExtendedDataItems{ - disposition: uint32(disposition), - status: status, - callstack: callstack, - items: make([]EventHeaderExtendedDataItem, 3), - } + f := getFileExtendedDataItems() + f.disposition, f.status, f.callstack = uint32(disposition), status, callstack f.items[0] = EventHeaderExtendedDataItem{ ExtType: ExtTypeDisposition, @@ -614,6 +610,8 @@ func AppendEventHeaderFileExtendedDataItems(r *EventRecord, disposition uint64, uintptr(unsafe.Pointer(&f.callstack[0])), ), } + } else { + f.items[2] = EventHeaderExtendedDataItem{} // clear stale entry from a prior pooled use } r.ExtendedDataCount = uint16(len(f.items)) @@ -705,15 +703,19 @@ func (e *EventRecord) ID() uint { return id } -// Copy makes a copy of this event record and returns the +// Clone makes a copy of this event record and returns the // copy itself and the event buffer. The buffer must outlive -// the event record instance. -func (r *EventRecord) Copy() (*EventRecord, []byte) { - c := *r - buf := make([]byte, r.BufferLen) +// the event record instance. Both are drawn from pools. +// Release with ReleaseRecord once fully done with the copy. +func (r *EventRecord) Clone() (*EventRecord, []byte) { + c := getEventRecord() + *c = *r + + buf := getBuf(int(r.BufferLen)) copy(buf, unsafe.Slice((*byte)(unsafe.Pointer(r.Buffer)), r.BufferLen)) c.Buffer = uintptr(unsafe.Pointer(&buf[0])) - return &c, buf + + return c, buf } // ReadByte reads the byte from the buffer at the specified offset. @@ -857,6 +859,14 @@ func (e *EventRecord) ConsumeUTF16String(offset uint16) string { return utf16.Decode(s[:len(s)/2-1]) } +// ConsumeRawUTF16String reads the byte slice for the UTF16-encoded string. +func (e *EventRecord) ConsumeRawUTF16String(offset uint16) []uint16 { + if offset > e.BufferLen { + return nil + } + return (*[1<<30 - 1]uint16)(unsafe.Pointer(e.Buffer + uintptr(offset)))[: e.BufferLen-offset : e.BufferLen-offset] +} + // ReadSID reads the security identifier from the event buffer. func (e *EventRecord) ReadSID(offset uint16, isWbemSid bool) ([]byte, uint16) { // this is a Security Token which can be null and takes 4 bytes. @@ -885,18 +895,28 @@ func (e *EventRecord) ReadSID(offset uint16, isWbemSid bool) ([]byte, uint16) { // ReadCallstack reads callstack return addresses at the given offset. func (e *EventRecord) ReadCallstack(offset uint16) []va.Address { + return e.ReadCallstackInto(offset, nil) +} + +// ReadCallstackInto reads callstack return addresses at the given offset, +// appending into v. If v has sufficient capacity, no allocation occurs. +func (e *EventRecord) ReadCallstackInto(offset uint16, v va.Callstack) []va.Address { var n uint16 m := offset - frames := (e.BufferLen - offset) / 8 - callstack := make([]va.Address, frames) + + if v != nil { + v = v[:0] + } else { + v = make([]va.Address, 0, frames) + } + for n < frames { - callstack[n] = va.Address(e.ReadUint64(m)) + v = append(v, va.Address(e.ReadUint64(m))) m += 8 n++ } - - return callstack + return v } // EventExtendedItemStackTrace64 defines a call stack on a 64-bit machine. diff --git a/pkg/util/va/address.go b/pkg/util/va/address.go index bbbf508b7..fbaadaf2f 100644 --- a/pkg/util/va/address.go +++ b/pkg/util/va/address.go @@ -18,11 +18,26 @@ package va -import "strconv" +import ( + "strconv" + "sync" +) // Address represents the memory address type Address uint64 +// Callstack is the type alias for callstack return addresses +type Callstack []Address + +const callstackDepthHint = 60 + +var callstackPool = sync.Pool{ + New: func() any { + c := make(Callstack, 0, callstackDepthHint) + return &c + }, +} + // Hex returns the hexadecimal representation of the memory address. func (a Address) String() string { return strconv.FormatUint(uint64(a), 16) } func (a Address) Uint64() uint64 { return uint64(a) } @@ -44,3 +59,15 @@ func (a Address) Dec(offset uint64) Address { // InSystemRange determines if this address is in the system address space range. // The kernel preferentially uses these two ranges to load DLLs at shared addresses. func (a Address) InSystemRange() bool { return a >= 0xfffff80000000000 && a < 0xffffffffffffffff } + +func GetCallstack() Callstack { + return (*callstackPool.Get().(*Callstack))[:0] +} + +func (c Callstack) ReleasePool() { + if cap(c) > callstackDepthHint*4 { + return // drop outlier-sized buffers rather than pooling them permanently + } + c = c[:0] + callstackPool.Put(&c) +} From 1b4c41314913d4b0d5b7426b2866c52bb84148c6 Mon Sep 17 00:00:00 2001 From: rabbitstack Date: Fri, 17 Jul 2026 19:21:14 +0200 Subject: [PATCH 4/4] perf(approvers): Implement path string interning and object pooling --- internal/etw/approvers/approver.go | 16 ++-- internal/etw/approvers/approver_test.go | 22 ++--- internal/etw/approvers/fs.go | 115 +++++++++++++++--------- internal/etw/approvers/fs_test.go | 2 +- internal/etw/approvers/registry.go | 71 +++++++++++---- internal/etw/consumer.go | 2 +- 6 files changed, 148 insertions(+), 80 deletions(-) diff --git a/internal/etw/approvers/approver.go b/internal/etw/approvers/approver.go index 380354abc..8a7de5b04 100644 --- a/internal/etw/approvers/approver.go +++ b/internal/etw/approvers/approver.go @@ -22,7 +22,6 @@ import ( "path/filepath" "strings" - "github.com/rabbitstack/fibratus/internal/etw/processors" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/filter/ql" "github.com/rabbitstack/fibratus/pkg/ps" @@ -46,12 +45,12 @@ type Approvers struct { } // New creates a new approvers set. -func New(psnap ps.Snapshotter, r *config.RulesCompileResult, processors *processors.Chain) Approvers { +func New(psnap ps.Snapshotter, r *config.RulesCompileResult) Approvers { p := Approvers{ approvers: make([]Approver, 0), } - p.fs = newFSApprover(r, processors).(*fs) + p.fs = newFSApprover(r).(*fs) p.approvers = append(p.approvers, p.fs) if r != nil { @@ -118,26 +117,25 @@ func (p *approver) approveExecutable(exe string) bool { } func (*approver) matchPredicate(m map[string][]string, v string) bool { - s := strings.ToLower(v) for op, patterns := range m { for _, pattern := range patterns { switch op { case ql.IMatches.String(): - if wildcard.Match(pattern, s) { + if wildcard.Match(pattern, v, false) { return true } case ql.IContains.String(): - if strings.Contains(s, pattern) { + if strings.Contains(strings.ToLower(v), pattern) { return true } case ql.Eq.String(), ql.In.String(): return v == pattern case ql.IEq.String(), ql.IIn.String(): - return s == pattern + return strings.EqualFold(v, pattern) case ql.IStartswith.String(): - return strings.HasPrefix(s, pattern) + return strings.HasPrefix(strings.ToLower(v), pattern) case ql.IEndswith.String(): - return strings.HasSuffix(s, pattern) + return strings.HasSuffix(strings.ToLower(v), pattern) } } } diff --git a/internal/etw/approvers/approver_test.go b/internal/etw/approvers/approver_test.go index e010e8ddd..1f2252dea 100644 --- a/internal/etw/approvers/approver_test.go +++ b/internal/etw/approvers/approver_test.go @@ -65,7 +65,7 @@ func TestApproversChainPassesEnrichedRecord(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) r := &etw.EventRecord{} r.Header.ProviderID = etw.WindowsKernelProcessGUID @@ -95,7 +95,7 @@ func TestApproversFirstApproverRejectsShortCircuits(t *testing.T) { func TestApproversCleanupDelegatesToFS(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) // enqueue a CreateFile cr := createFileRecord(t, createBuf) @@ -110,7 +110,7 @@ func TestApproversCleanupDelegatesToFS(t *testing.T) { func TestApproversFileEventFullFlow(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) // CreateFile is suppressed and stored cr := createFileRecord(t, createBuf) @@ -137,7 +137,7 @@ func TestApproversFileEventWithRulesApproved(t *testing.T) { } psnap := &ps.SnapshotterMock{} - p := New(psnap, rules, nil) + p := New(psnap, rules) cr := createFileRecord(t, createBuf) p.Approve(cr) @@ -167,7 +167,7 @@ func TestApproversFileEventWithRulesRejected(t *testing.T) { } psnap := &ps.SnapshotterMock{} - p := New(psnap, rules, nil) + p := New(psnap, rules) cr := createFileRecord(t, createBuf) p.Approve(cr) @@ -181,7 +181,7 @@ func TestApproversFileEventWithRulesRejected(t *testing.T) { func TestApproversRegistryEventNoRulesApprovesAll(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) _, approved := p.Approve(r) @@ -198,7 +198,7 @@ func TestApproversRegistryEventWithRulesApproved(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) k := makeRecord(event.RegistryEventGUID, event.RegCreateKCBID, 0, regCreateKCBBuf) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) @@ -221,7 +221,7 @@ func TestApproversRegistryEventWithRulesRejected(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) k := makeRecord(event.RegistryEventGUID, event.RegCreateKCBID, 0, regCreateKCBBuf) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) @@ -234,7 +234,7 @@ func TestApproversRegistryEventWithRulesRejected(t *testing.T) { func TestApproversProcEventNoRulesApprovesAll(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) buf := make([]byte, 4) *(*uint32)(unsafe.Pointer(&buf[0])) = uint32(1234) @@ -256,7 +256,7 @@ func TestApproversProcEventWithRulesApproved(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) const pid = uint32(1234) psnap.On("Find", pid).Return(true, &pstypes.PS{ @@ -281,7 +281,7 @@ func TestApproversProcEventWithRulesRejected(t *testing.T) { }, }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) const pid = uint32(5678) psnap.On("Find", pid).Return(true, &pstypes.PS{ diff --git a/internal/etw/approvers/fs.go b/internal/etw/approvers/fs.go index 083a23353..093bb08c7 100644 --- a/internal/etw/approvers/fs.go +++ b/internal/etw/approvers/fs.go @@ -20,13 +20,17 @@ package approvers import ( "expvar" + "time" + "unsafe" - "github.com/rabbitstack/fibratus/internal/etw/processors" + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/event" devmapper "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/sys/etw" + "github.com/rabbitstack/fibratus/pkg/util/utf16" "github.com/rabbitstack/fibratus/pkg/util/va" + "github.com/zeebo/xxh3" "golang.org/x/sys/windows" ) @@ -40,12 +44,31 @@ var ( // irp acts as a scratch area for the pending IRP request // that is used as a signal to promote the file operation. // The memory buffer backing the event record must outlive -// event processors scope. +// event processors scope, along with the extended data items +// and the event callstack. type irp struct { - rec *etw.EventRecord - buf []byte // keeps the data buffer alive + rec *etw.EventRecord // keeps the original even record items *etw.FileExtendedDataItems // keeps the extended data items alive - callstack []va.Address // CreateFile stack return addresses + buf []byte // keeps the data buffer alive + callstack va.Callstack // CreateFile stack return addresses +} + +func (p *irp) storeExtendedDataItems(disposition uint64, status uint32) { + p.items = etw.AppendEventHeaderFileExtendedDataItems(p.rec, disposition, status, p.callstack) +} + +func (p *irp) storeCallstack(r *etw.EventRecord) { + p.callstack = r.ReadCallstackInto(16, va.GetCallstack()) +} + +func (p *irp) releasePools() { + p.rec.ReleasePool() + if p.items != nil { + p.items.ReleasePool() + } + if p.callstack != nil { + p.callstack.ReleasePool() + } } // fs is the file system approver that accepts or discards @@ -56,18 +79,24 @@ type fs struct { irps map[uint64]irp - processors *processors.Chain - approvers []func(string) bool + approvers []func(string) bool + + pathLRUCache *expirable.LRU[uint64, string] // interned file paths } -func newFSApprover(r *config.RulesCompileResult, processors *processors.Chain) Approver { +func hashFilePath(u []uint16) uint64 { + b := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(u))), len(u)) + return xxh3.Hash(b) +} + +func newFSApprover(r *config.RulesCompileResult) Approver { fs := &fs{ approver: approver{ r: r, }, - approvers: make([]func(string) bool, 0), - processors: processors, - irps: make(map[uint64]irp), + approvers: make([]func(string) bool, 0), + irps: make(map[uint64]irp, 120), + pathLRUCache: expirable.NewLRU[uint64, string](2500, nil, time.Minute*5), } if r != nil && len(r.Approvers.Paths) > 0 { @@ -89,24 +118,26 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { return r, true } - // enqueue in flight CreateFile event + // clone and enqueue in flight CreateFile event if r.Header.EventDescriptor.Opcode == event.CreateFileID { - rec, buf := r.Copy() - f.irps[r.ReadUint64(0)] = irp{rec: rec, buf: buf} + i := r.ReadUint64(0) + rec, buf := r.Clone() + f.irps[i] = irp{rec: rec, buf: buf} return r, false } - // decorate the pending CreateFile with the return addresses - // obtained from the StackWalk event by correlating the timestamp + // associate the callstack return addresses to the CreateFile event + // obtained from the StackWalk event by event timestamp correlation if isStackwalk { ts := r.ReadUint64(0) for i, rec := range f.irps { - if rec.rec.Header.Timestamp == ts { - irp := f.irps[i] - irp.callstack = r.ReadCallstack(16) - f.irps[i] = irp - return r, false + if rec.rec.Header.Timestamp != ts { + continue } + irp := f.irps[i] + irp.storeCallstack(r) + f.irps[i] = irp + return r, false } return r, true } @@ -138,33 +169,34 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { // or the rules compilation result is not present // we'll allow events flow downstream processors if f.r == nil || disposition != windows.FILE_OPEN { - irp.items = etw.AppendEventHeaderFileExtendedDataItems(rec, disposition, status, irp.callstack) + irp.storeExtendedDataItems(disposition, status) f.irps[i] = irp return rec, true } + s := rec.ConsumeRawUTF16String(32) + h := hashFilePath(s) + path, ok := f.pathLRUCache.Get(h) + if !ok { + n := utf16.Decode(s[:len(s)/2-1]) + path = devmapper.GetDevMapper().Convert(n) + f.pathLRUCache.Add(h, path) + } + // evaluate against available approvers - var approved bool - path := devmapper.GetDevMapper().Convert(rec.ConsumeUTF16String(32)) for _, approver := range f.approvers { if approver(path) { - approved = true - break + fsApproverApprovals.Add(1) + irp.storeExtendedDataItems(disposition, status) + f.irps[i] = irp + return rec, true } } - if !approved { - // the event is rejected by approvers. Make sure to - // evict the enqueded StackWalk event produced by the - // CreateFile operation - delete(f.irps, i) - fsApproverRejections.Add(1) - return rec, false - } - - fsApproverApprovals.Add(1) - irp.items = etw.AppendEventHeaderFileExtendedDataItems(rec, disposition, status, irp.callstack) - f.irps[i] = irp - return rec, true + // the event is rejected by approvers + irp.releasePools() + delete(f.irps, i) + fsApproverRejections.Add(1) + return rec, false } return r, true @@ -173,6 +205,9 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { func (f *fs) cleanup(r *etw.EventRecord) { if r.Header.ProviderID == event.FileEventGUID && r.Header.EventDescriptor.Opcode == event.CreateFileID { i := r.ReadUint64(0) - delete(f.irps, i) + if irp, ok := f.irps[i]; ok { + irp.releasePools() + delete(f.irps, i) + } } } diff --git a/internal/etw/approvers/fs_test.go b/internal/etw/approvers/fs_test.go index 5e5a681ac..e98bf48a6 100644 --- a/internal/etw/approvers/fs_test.go +++ b/internal/etw/approvers/fs_test.go @@ -157,7 +157,7 @@ func buildMatchingFileOpEnd(t *testing.T, createBuf []byte, disposition uint64) } func newTestFSApprover(r *config.RulesCompileResult) *fs { - return newFSApprover(r, nil).(*fs) + return newFSApprover(r).(*fs) } func TestFSApproverNonFileEvent(t *testing.T) { diff --git a/internal/etw/approvers/registry.go b/internal/etw/approvers/registry.go index ade474d52..defe3fd22 100644 --- a/internal/etw/approvers/registry.go +++ b/internal/etw/approvers/registry.go @@ -20,7 +20,14 @@ package approvers import ( "expvar" + "time" + "unsafe" + "github.com/zeebo/xxh3" + + "github.com/rabbitstack/fibratus/pkg/util/utf16" + + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/sys/etw" @@ -37,6 +44,8 @@ var ( type registry struct { approver kcbs map[uint64]string + + pathLRUCache *expirable.LRU[uint64, string] // interned KCB paths } func newRegistryApprover(r *config.RulesCompileResult) Approver { @@ -44,8 +53,22 @@ func newRegistryApprover(r *config.RulesCompileResult) Approver { approver: approver{ r: r, }, - kcbs: make(map[uint64]string), + kcbs: make(map[uint64]string, 5000), + pathLRUCache: expirable.NewLRU[uint64, string](1000, nil, time.Minute*5), + } +} + +func hashKeyPath(r string, u []uint16) uint64 { + var h xxh3.Hasher + + b := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(u))), len(u)) + _, _ = h.Write(b) + if r != "" { + k := unsafe.Slice(unsafe.StringData(r), len(r)) + _, _ = h.Write(k) } + + return h.Sum64() } func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { @@ -58,11 +81,16 @@ func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { // keep the state of allocated key control blocks // to be able to derive the full registry path if id == event.RegKCBRundownID || id == event.RegCreateKCBID { - r.kcbs[rec.ReadUint64(16)] = rec.ConsumeUTF16String(24) + kcb := rec.ReadUint64(16) + path := rec.ConsumeUTF16String(24) + r.kcbs[kcb] = path return rec, true } + // remove kcb mapping if id == event.RegDeleteKCBID { - delete(r.kcbs, rec.ReadUint64(16)) + kcb := rec.ReadUint64(16) + r.pathLRUCache.Purge() + delete(r.kcbs, kcb) return rec, true } @@ -71,24 +99,31 @@ func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { return rec, true } - // lookup KCB map to check if the event - // KCB object address references a key - // we can use to reconstruct the full - // registry path + // calculate the hash and check against LRU cache kcb := rec.ReadUint64(16) - path := rec.ConsumeUTF16String(24) - if kcb != 0 { - path = key.ConcatPaths(r.kcbs[kcb], path) - } + s := rec.ConsumeRawUTF16String(24) + h := hashKeyPath(r.kcbs[kcb], s) + path, ok := r.pathLRUCache.Get(h) + if !ok { + // lookup KCB map to check if the event + // KCB object address references a key + // we can use to reconstruct the full + // registry path + path = utf16.Decode(s[:len(s)/2-1]) + if kcb != 0 { + path = key.ConcatPaths(r.kcbs[kcb], path) + } - rootkey, subkey := key.Format(path) - if rootkey != key.Invalid { - root := rootkey.String() - if subkey != "" { - path = key.ConcatPaths(root, subkey) - } else { - path = root + rootkey, subkey := key.Format(path) + if rootkey != key.Invalid { + root := rootkey.String() + if subkey != "" { + path = key.ConcatPaths(root, subkey) + } else { + path = root + } } + r.pathLRUCache.Add(h, path) } if r.approveKey(path) { diff --git a/internal/etw/consumer.go b/internal/etw/consumer.go index f2f599d54..b63634970 100644 --- a/internal/etw/consumer.go +++ b/internal/etw/consumer.go @@ -60,7 +60,7 @@ func NewConsumer( processors: processors, psnap: psnap, config: config, - approvers: approvers.New(psnap, r, processors), + approvers: approvers.New(psnap, r), } }