diff --git a/go.mod b/go.mod index 839eccae8..8510793f0 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/enescakir/emoji v1.0.0 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/hashicorp/go-version v1.2.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hillu/go-yara/v4 v4.2.4 github.com/jedib0t/go-pretty/v6 v6.2.1 github.com/lithammer/fuzzysearch v1.1.2 @@ -33,6 +34,7 @@ require ( github.com/valyala/gozstd v1.11.0 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yuin/goldmark v1.5.2 + github.com/zeebo/xxh3 v1.1.0 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 golang.org/x/arch v0.6.0 golang.org/x/sys v0.31.0 @@ -41,11 +43,12 @@ require ( gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 - www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc ) require ( github.com/BurntSushi/toml v0.4.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d // indirect @@ -54,6 +57,7 @@ require ( golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect honnef.co/go/tools v0.3.2 // indirect ) diff --git a/go.sum b/go.sum index c11b25fdb..dc48d1a7d 100644 --- a/go.sum +++ b/go.sum @@ -11,12 +11,6 @@ github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFP github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= -github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= -github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= -github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= -github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= @@ -40,6 +34,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -91,6 +86,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hillu/go-yara/v4 v4.2.4 h1:r3KB1XV+h6q+N8bvK6/gLpxAVcd6baYzmOSYHzNo9QQ= @@ -112,14 +109,18 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lithammer/fuzzysearch v1.1.2 h1:ePUtm14xKxbpCxozcFbIDRtvANxnVnE+RKpJUqkr2gA= github.com/lithammer/fuzzysearch v1.1.2/go.mod h1:v6tYW/9kpfV6LNcweXdSjQsfCku/1M/oObmSox1fzP8= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= @@ -151,6 +152,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -175,16 +177,13 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/saferwall/pe v1.5.6 h1:DrRLnoQFxHWJ5lJUmrH7X2L0xeUu6SUS95Dc61eW2Yc= github.com/saferwall/pe v1.5.6/go.mod h1:mJx+PuptmNpoPFBNhWs/uDMFL/kTHVZIkg0d4OUJFbQ= -github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= -github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d h1:RQqyEogx5J6wPdoxqL132b100j8KjcVHO1c0KLRoIhc= github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d/go.mod h1:PegD7EVqlN88z7TpCqH92hHP+GBpfomGCCnw1PFtNOA= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -251,6 +250,10 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU= github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= @@ -303,8 +306,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -385,5 +386,3 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= -www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc h1:eeL+RUEGr6/lYL8hJEbvugrF88I6W4pBaVtFa1falj4= -www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= diff --git a/internal/etw/approvers/approver.go b/internal/etw/approvers/approver.go index 380354abc..8a7de5b04 100644 --- a/internal/etw/approvers/approver.go +++ b/internal/etw/approvers/approver.go @@ -22,7 +22,6 @@ import ( "path/filepath" "strings" - "github.com/rabbitstack/fibratus/internal/etw/processors" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/filter/ql" "github.com/rabbitstack/fibratus/pkg/ps" @@ -46,12 +45,12 @@ type Approvers struct { } // New creates a new approvers set. -func New(psnap ps.Snapshotter, r *config.RulesCompileResult, processors *processors.Chain) Approvers { +func New(psnap ps.Snapshotter, r *config.RulesCompileResult) Approvers { p := Approvers{ approvers: make([]Approver, 0), } - p.fs = newFSApprover(r, processors).(*fs) + p.fs = newFSApprover(r).(*fs) p.approvers = append(p.approvers, p.fs) if r != nil { @@ -118,26 +117,25 @@ func (p *approver) approveExecutable(exe string) bool { } func (*approver) matchPredicate(m map[string][]string, v string) bool { - s := strings.ToLower(v) for op, patterns := range m { for _, pattern := range patterns { switch op { case ql.IMatches.String(): - if wildcard.Match(pattern, s) { + if wildcard.Match(pattern, v, false) { return true } case ql.IContains.String(): - if strings.Contains(s, pattern) { + if strings.Contains(strings.ToLower(v), pattern) { return true } case ql.Eq.String(), ql.In.String(): return v == pattern case ql.IEq.String(), ql.IIn.String(): - return s == pattern + return strings.EqualFold(v, pattern) case ql.IStartswith.String(): - return strings.HasPrefix(s, pattern) + return strings.HasPrefix(strings.ToLower(v), pattern) case ql.IEndswith.String(): - return strings.HasSuffix(s, pattern) + return strings.HasSuffix(strings.ToLower(v), pattern) } } } diff --git a/internal/etw/approvers/approver_test.go b/internal/etw/approvers/approver_test.go index e010e8ddd..1f2252dea 100644 --- a/internal/etw/approvers/approver_test.go +++ b/internal/etw/approvers/approver_test.go @@ -65,7 +65,7 @@ func TestApproversChainPassesEnrichedRecord(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) r := &etw.EventRecord{} r.Header.ProviderID = etw.WindowsKernelProcessGUID @@ -95,7 +95,7 @@ func TestApproversFirstApproverRejectsShortCircuits(t *testing.T) { func TestApproversCleanupDelegatesToFS(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) // enqueue a CreateFile cr := createFileRecord(t, createBuf) @@ -110,7 +110,7 @@ func TestApproversCleanupDelegatesToFS(t *testing.T) { func TestApproversFileEventFullFlow(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) // CreateFile is suppressed and stored cr := createFileRecord(t, createBuf) @@ -137,7 +137,7 @@ func TestApproversFileEventWithRulesApproved(t *testing.T) { } psnap := &ps.SnapshotterMock{} - p := New(psnap, rules, nil) + p := New(psnap, rules) cr := createFileRecord(t, createBuf) p.Approve(cr) @@ -167,7 +167,7 @@ func TestApproversFileEventWithRulesRejected(t *testing.T) { } psnap := &ps.SnapshotterMock{} - p := New(psnap, rules, nil) + p := New(psnap, rules) cr := createFileRecord(t, createBuf) p.Approve(cr) @@ -181,7 +181,7 @@ func TestApproversFileEventWithRulesRejected(t *testing.T) { func TestApproversRegistryEventNoRulesApprovesAll(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) _, approved := p.Approve(r) @@ -198,7 +198,7 @@ func TestApproversRegistryEventWithRulesApproved(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) k := makeRecord(event.RegistryEventGUID, event.RegCreateKCBID, 0, regCreateKCBBuf) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) @@ -221,7 +221,7 @@ func TestApproversRegistryEventWithRulesRejected(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) k := makeRecord(event.RegistryEventGUID, event.RegCreateKCBID, 0, regCreateKCBBuf) r := makeRecord(event.RegistryEventGUID, event.RegOpenKeyID, 0, regOpenKeyBuf) @@ -234,7 +234,7 @@ func TestApproversRegistryEventWithRulesRejected(t *testing.T) { func TestApproversProcEventNoRulesApprovesAll(t *testing.T) { psnap := &ps.SnapshotterMock{} - p := New(psnap, nil, nil) + p := New(psnap, nil) buf := make([]byte, 4) *(*uint32)(unsafe.Pointer(&buf[0])) = uint32(1234) @@ -256,7 +256,7 @@ func TestApproversProcEventWithRulesApproved(t *testing.T) { }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) const pid = uint32(1234) psnap.On("Find", pid).Return(true, &pstypes.PS{ @@ -281,7 +281,7 @@ func TestApproversProcEventWithRulesRejected(t *testing.T) { }, }, } - p := New(psnap, rules, nil) + p := New(psnap, rules) const pid = uint32(5678) psnap.On("Find", pid).Return(true, &pstypes.PS{ diff --git a/internal/etw/approvers/fs.go b/internal/etw/approvers/fs.go index 083a23353..093bb08c7 100644 --- a/internal/etw/approvers/fs.go +++ b/internal/etw/approvers/fs.go @@ -20,13 +20,17 @@ package approvers import ( "expvar" + "time" + "unsafe" - "github.com/rabbitstack/fibratus/internal/etw/processors" + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/event" devmapper "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/sys/etw" + "github.com/rabbitstack/fibratus/pkg/util/utf16" "github.com/rabbitstack/fibratus/pkg/util/va" + "github.com/zeebo/xxh3" "golang.org/x/sys/windows" ) @@ -40,12 +44,31 @@ var ( // irp acts as a scratch area for the pending IRP request // that is used as a signal to promote the file operation. // The memory buffer backing the event record must outlive -// event processors scope. +// event processors scope, along with the extended data items +// and the event callstack. type irp struct { - rec *etw.EventRecord - buf []byte // keeps the data buffer alive + rec *etw.EventRecord // keeps the original even record items *etw.FileExtendedDataItems // keeps the extended data items alive - callstack []va.Address // CreateFile stack return addresses + buf []byte // keeps the data buffer alive + callstack va.Callstack // CreateFile stack return addresses +} + +func (p *irp) storeExtendedDataItems(disposition uint64, status uint32) { + p.items = etw.AppendEventHeaderFileExtendedDataItems(p.rec, disposition, status, p.callstack) +} + +func (p *irp) storeCallstack(r *etw.EventRecord) { + p.callstack = r.ReadCallstackInto(16, va.GetCallstack()) +} + +func (p *irp) releasePools() { + p.rec.ReleasePool() + if p.items != nil { + p.items.ReleasePool() + } + if p.callstack != nil { + p.callstack.ReleasePool() + } } // fs is the file system approver that accepts or discards @@ -56,18 +79,24 @@ type fs struct { irps map[uint64]irp - processors *processors.Chain - approvers []func(string) bool + approvers []func(string) bool + + pathLRUCache *expirable.LRU[uint64, string] // interned file paths } -func newFSApprover(r *config.RulesCompileResult, processors *processors.Chain) Approver { +func hashFilePath(u []uint16) uint64 { + b := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(u))), len(u)) + return xxh3.Hash(b) +} + +func newFSApprover(r *config.RulesCompileResult) Approver { fs := &fs{ approver: approver{ r: r, }, - approvers: make([]func(string) bool, 0), - processors: processors, - irps: make(map[uint64]irp), + approvers: make([]func(string) bool, 0), + irps: make(map[uint64]irp, 120), + pathLRUCache: expirable.NewLRU[uint64, string](2500, nil, time.Minute*5), } if r != nil && len(r.Approvers.Paths) > 0 { @@ -89,24 +118,26 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { return r, true } - // enqueue in flight CreateFile event + // clone and enqueue in flight CreateFile event if r.Header.EventDescriptor.Opcode == event.CreateFileID { - rec, buf := r.Copy() - f.irps[r.ReadUint64(0)] = irp{rec: rec, buf: buf} + i := r.ReadUint64(0) + rec, buf := r.Clone() + f.irps[i] = irp{rec: rec, buf: buf} return r, false } - // decorate the pending CreateFile with the return addresses - // obtained from the StackWalk event by correlating the timestamp + // associate the callstack return addresses to the CreateFile event + // obtained from the StackWalk event by event timestamp correlation if isStackwalk { ts := r.ReadUint64(0) for i, rec := range f.irps { - if rec.rec.Header.Timestamp == ts { - irp := f.irps[i] - irp.callstack = r.ReadCallstack(16) - f.irps[i] = irp - return r, false + if rec.rec.Header.Timestamp != ts { + continue } + irp := f.irps[i] + irp.storeCallstack(r) + f.irps[i] = irp + return r, false } return r, true } @@ -138,33 +169,34 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { // or the rules compilation result is not present // we'll allow events flow downstream processors if f.r == nil || disposition != windows.FILE_OPEN { - irp.items = etw.AppendEventHeaderFileExtendedDataItems(rec, disposition, status, irp.callstack) + irp.storeExtendedDataItems(disposition, status) f.irps[i] = irp return rec, true } + s := rec.ConsumeRawUTF16String(32) + h := hashFilePath(s) + path, ok := f.pathLRUCache.Get(h) + if !ok { + n := utf16.Decode(s[:len(s)/2-1]) + path = devmapper.GetDevMapper().Convert(n) + f.pathLRUCache.Add(h, path) + } + // evaluate against available approvers - var approved bool - path := devmapper.GetDevMapper().Convert(rec.ConsumeUTF16String(32)) for _, approver := range f.approvers { if approver(path) { - approved = true - break + fsApproverApprovals.Add(1) + irp.storeExtendedDataItems(disposition, status) + f.irps[i] = irp + return rec, true } } - if !approved { - // the event is rejected by approvers. Make sure to - // evict the enqueded StackWalk event produced by the - // CreateFile operation - delete(f.irps, i) - fsApproverRejections.Add(1) - return rec, false - } - - fsApproverApprovals.Add(1) - irp.items = etw.AppendEventHeaderFileExtendedDataItems(rec, disposition, status, irp.callstack) - f.irps[i] = irp - return rec, true + // the event is rejected by approvers + irp.releasePools() + delete(f.irps, i) + fsApproverRejections.Add(1) + return rec, false } return r, true @@ -173,6 +205,9 @@ func (f *fs) Approve(r *etw.EventRecord) (*etw.EventRecord, bool) { func (f *fs) cleanup(r *etw.EventRecord) { if r.Header.ProviderID == event.FileEventGUID && r.Header.EventDescriptor.Opcode == event.CreateFileID { i := r.ReadUint64(0) - delete(f.irps, i) + if irp, ok := f.irps[i]; ok { + irp.releasePools() + delete(f.irps, i) + } } } diff --git a/internal/etw/approvers/fs_test.go b/internal/etw/approvers/fs_test.go index 5e5a681ac..e98bf48a6 100644 --- a/internal/etw/approvers/fs_test.go +++ b/internal/etw/approvers/fs_test.go @@ -157,7 +157,7 @@ func buildMatchingFileOpEnd(t *testing.T, createBuf []byte, disposition uint64) } func newTestFSApprover(r *config.RulesCompileResult) *fs { - return newFSApprover(r, nil).(*fs) + return newFSApprover(r).(*fs) } func TestFSApproverNonFileEvent(t *testing.T) { diff --git a/internal/etw/approvers/registry.go b/internal/etw/approvers/registry.go index ade474d52..defe3fd22 100644 --- a/internal/etw/approvers/registry.go +++ b/internal/etw/approvers/registry.go @@ -20,7 +20,14 @@ package approvers import ( "expvar" + "time" + "unsafe" + "github.com/zeebo/xxh3" + + "github.com/rabbitstack/fibratus/pkg/util/utf16" + + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/sys/etw" @@ -37,6 +44,8 @@ var ( type registry struct { approver kcbs map[uint64]string + + pathLRUCache *expirable.LRU[uint64, string] // interned KCB paths } func newRegistryApprover(r *config.RulesCompileResult) Approver { @@ -44,8 +53,22 @@ func newRegistryApprover(r *config.RulesCompileResult) Approver { approver: approver{ r: r, }, - kcbs: make(map[uint64]string), + kcbs: make(map[uint64]string, 5000), + pathLRUCache: expirable.NewLRU[uint64, string](1000, nil, time.Minute*5), + } +} + +func hashKeyPath(r string, u []uint16) uint64 { + var h xxh3.Hasher + + b := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(u))), len(u)) + _, _ = h.Write(b) + if r != "" { + k := unsafe.Slice(unsafe.StringData(r), len(r)) + _, _ = h.Write(k) } + + return h.Sum64() } func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { @@ -58,11 +81,16 @@ func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { // keep the state of allocated key control blocks // to be able to derive the full registry path if id == event.RegKCBRundownID || id == event.RegCreateKCBID { - r.kcbs[rec.ReadUint64(16)] = rec.ConsumeUTF16String(24) + kcb := rec.ReadUint64(16) + path := rec.ConsumeUTF16String(24) + r.kcbs[kcb] = path return rec, true } + // remove kcb mapping if id == event.RegDeleteKCBID { - delete(r.kcbs, rec.ReadUint64(16)) + kcb := rec.ReadUint64(16) + r.pathLRUCache.Purge() + delete(r.kcbs, kcb) return rec, true } @@ -71,24 +99,31 @@ func (r *registry) Approve(rec *etw.EventRecord) (*etw.EventRecord, bool) { return rec, true } - // lookup KCB map to check if the event - // KCB object address references a key - // we can use to reconstruct the full - // registry path + // calculate the hash and check against LRU cache kcb := rec.ReadUint64(16) - path := rec.ConsumeUTF16String(24) - if kcb != 0 { - path = key.ConcatPaths(r.kcbs[kcb], path) - } + s := rec.ConsumeRawUTF16String(24) + h := hashKeyPath(r.kcbs[kcb], s) + path, ok := r.pathLRUCache.Get(h) + if !ok { + // lookup KCB map to check if the event + // KCB object address references a key + // we can use to reconstruct the full + // registry path + path = utf16.Decode(s[:len(s)/2-1]) + if kcb != 0 { + path = key.ConcatPaths(r.kcbs[kcb], path) + } - rootkey, subkey := key.Format(path) - if rootkey != key.Invalid { - root := rootkey.String() - if subkey != "" { - path = key.ConcatPaths(root, subkey) - } else { - path = root + rootkey, subkey := key.Format(path) + if rootkey != key.Invalid { + root := rootkey.String() + if subkey != "" { + path = key.ConcatPaths(root, subkey) + } else { + path = root + } } + r.pathLRUCache.Add(h, path) } if r.approveKey(path) { diff --git a/internal/etw/consumer.go b/internal/etw/consumer.go index f2f599d54..b63634970 100644 --- a/internal/etw/consumer.go +++ b/internal/etw/consumer.go @@ -60,7 +60,7 @@ func NewConsumer( processors: processors, psnap: psnap, config: config, - approvers: approvers.New(psnap, r, processors), + approvers: approvers.New(psnap, r), } } diff --git a/pkg/filter/ql/ast.go b/pkg/filter/ql/ast.go index 124dfd542..aec705db2 100644 --- a/pkg/filter/ql/ast.go +++ b/pkg/filter/ql/ast.go @@ -30,18 +30,6 @@ import ( "github.com/rabbitstack/fibratus/pkg/util/wildcard" ) -// loweredWildcards contains a mapping between original and lower-case wildcard patterns -var loweredWildcards = map[string]string{} - -func toLowerWildcard(p string) string { - if s, ok := loweredWildcards[p]; ok { - return s - } - s := strings.ToLower(p) - loweredWildcards[p] = s - return s -} - // Eval evaluates expr against a map that contains the field values. func Eval(expr Expr, m map[string]interface{}, useFuncValuer bool) bool { var eval ValuerEval @@ -961,10 +949,10 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { case Matches: switch rhs := rhs.(type) { case string: - return wildcard.Match(rhs, lhs) + return wildcard.Match(rhs, lhs, true) case []string: for _, pat := range rhs { - if wildcard.Match(pat, lhs) { + if wildcard.Match(pat, lhs, true) { return true } } @@ -973,13 +961,12 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { return false } case IMatches: - v := strings.ToLower(lhs) switch rhs := rhs.(type) { case string: - return wildcard.Match(toLowerWildcard(rhs), v) + return wildcard.Match(rhs, lhs, false) case []string: for _, pat := range rhs { - if wildcard.Match(toLowerWildcard(pat), v) { + if wildcard.Match(pat, lhs, false) { return true } } @@ -1231,7 +1218,7 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { } for _, pat := range rhs { for _, val := range lhs { - if wildcard.Match(pat, val) { + if wildcard.Match(pat, val, true) { return true } } @@ -1244,7 +1231,7 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { } for _, pat := range rhs { for _, val := range lhs { - if wildcard.Match(toLowerWildcard(pat), strings.ToLower(val)) { + if wildcard.Match(pat, val, false) { return true } } diff --git a/pkg/filter/ql/functions/count.go b/pkg/filter/ql/functions/count.go index 1ed051659..e1cb8842a 100644 --- a/pkg/filter/ql/functions/count.go +++ b/pkg/filter/ql/functions/count.go @@ -50,11 +50,11 @@ func (f Count) Call(args []interface{}) (any, bool) { for _, ss := range substrings { switch caseInsensitive { case true: - if wildcard.Match(strings.ToLower(pattern), strings.ToLower(ss)) { + if wildcard.Match(pattern, ss, false) { count++ } case false: - if wildcard.Match(pattern, ss) { + if wildcard.Match(pattern, ss, true) { count++ } } @@ -63,11 +63,11 @@ func (f Count) Call(args []interface{}) (any, bool) { for _, i := range s { switch caseInsensitive { case true: - if wildcard.Match(strings.ToLower(pattern), strings.ToLower(i)) { + if wildcard.Match(pattern, i, false) { count++ } case false: - if wildcard.Match(pattern, i) { + if wildcard.Match(pattern, i, true) { count++ } } diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 003e6ad24..202d32bcc 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -84,7 +84,7 @@ var parserOpts = []pe.Option{ // from the Portable Executable headers. func GetFileInfo(path string) (*FileInfo, error) { for _, pat := range skippedPatterns { - if wildcard.Match(pat, path) { + if wildcard.Match(pat, path, false) { return nil, ErrSkippedFile(path) } } diff --git a/pkg/ps/types/types_windows.go b/pkg/ps/types/types_windows.go index f602ed5c7..b9a167066 100644 --- a/pkg/ps/types/types_windows.go +++ b/pkg/ps/types/types_windows.go @@ -321,7 +321,7 @@ func (ps *PS) IsAppinfoSvc() bool { // IsSvchost returns true if this is the Service Host process. func (ps *PS) IsSvchost() bool { - return wildcard.Match(`?:\windows\system32\svchost.exe`, strings.ToLower(ps.Exe)) + return wildcard.Match(`?:\windows\system32\svchost.exe`, ps.Exe, false) } // Thread stores metadata about a thread that's executing in process's address space. diff --git a/pkg/sys/etw/record_pool.go b/pkg/sys/etw/record_pool.go new file mode 100644 index 000000000..c56156af3 --- /dev/null +++ b/pkg/sys/etw/record_pool.go @@ -0,0 +1,85 @@ +//go:build windows +// +build windows + +/* + * Copyright 2019-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etw + +import ( + "sync" + "unsafe" +) + +const bufSizeHint = 512 + +var bufPool = sync.Pool{ + New: func() any { + b := make([]byte, 0, bufSizeHint) + return &b + }, +} + +var eventRecordPool = sync.Pool{ + New: func() any { return &EventRecord{} }, +} + +var fileExtDataPool = sync.Pool{ + New: func() any { return &FileExtendedDataItems{} }, +} + +func getBuf(n int) []byte { + bp := bufPool.Get().(*[]byte) + b := *bp + if cap(b) < n { + return make([]byte, n) // rare oversized event, pay the allocation once, don't repool it later + } + return b[:n] +} + +func releaseBuf(b []byte) { + if cap(b) > bufSizeHint*8 { + return // drop outlier-sized buffers instead of pooling them permanently + } + b = b[:0] + bufPool.Put(&b) +} + +func getEventRecord() *EventRecord { + return eventRecordPool.Get().(*EventRecord) +} + +func getFileExtendedDataItems() *FileExtendedDataItems { + return fileExtDataPool.Get().(*FileExtendedDataItems) +} + +func (f *FileExtendedDataItems) ReleasePool() { + f.callstack = nil // release reference, don't let pool retain a stale slice + fileExtDataPool.Put(f) +} + +// ReleasePool returns both the record and its backing buffer to their pools. +// Call only once the record is fully done being read by all downstream consumers. +func (r *EventRecord) ReleasePool() { + if r.Buffer != 0 && r.BufferLen > 0 { + buf := unsafe.Slice((*byte)(unsafe.Pointer(r.Buffer)), r.BufferLen) + releaseBuf(buf) + } + *r = EventRecord{} // zero every field before returning to pool + eventRecordPool.Put(r) +} diff --git a/pkg/sys/etw/types.go b/pkg/sys/etw/types.go index 2a217c920..ccb2d5d25 100644 --- a/pkg/sys/etw/types.go +++ b/pkg/sys/etw/types.go @@ -584,17 +584,13 @@ type FileExtendedDataItems struct { status uint32 disposition uint32 callstack []va.Address - items []EventHeaderExtendedDataItem + items [3]EventHeaderExtendedDataItem } // AppendEventHeaderFileExtendedDataItems appends custom file extendeed data items to the event record. func AppendEventHeaderFileExtendedDataItems(r *EventRecord, disposition uint64, status uint32, callstack []va.Address) *FileExtendedDataItems { - f := &FileExtendedDataItems{ - disposition: uint32(disposition), - status: status, - callstack: callstack, - items: make([]EventHeaderExtendedDataItem, 3), - } + f := getFileExtendedDataItems() + f.disposition, f.status, f.callstack = uint32(disposition), status, callstack f.items[0] = EventHeaderExtendedDataItem{ ExtType: ExtTypeDisposition, @@ -614,6 +610,8 @@ func AppendEventHeaderFileExtendedDataItems(r *EventRecord, disposition uint64, uintptr(unsafe.Pointer(&f.callstack[0])), ), } + } else { + f.items[2] = EventHeaderExtendedDataItem{} // clear stale entry from a prior pooled use } r.ExtendedDataCount = uint16(len(f.items)) @@ -705,15 +703,19 @@ func (e *EventRecord) ID() uint { return id } -// Copy makes a copy of this event record and returns the +// Clone makes a copy of this event record and returns the // copy itself and the event buffer. The buffer must outlive -// the event record instance. -func (r *EventRecord) Copy() (*EventRecord, []byte) { - c := *r - buf := make([]byte, r.BufferLen) +// the event record instance. Both are drawn from pools. +// Release with ReleaseRecord once fully done with the copy. +func (r *EventRecord) Clone() (*EventRecord, []byte) { + c := getEventRecord() + *c = *r + + buf := getBuf(int(r.BufferLen)) copy(buf, unsafe.Slice((*byte)(unsafe.Pointer(r.Buffer)), r.BufferLen)) c.Buffer = uintptr(unsafe.Pointer(&buf[0])) - return &c, buf + + return c, buf } // ReadByte reads the byte from the buffer at the specified offset. @@ -857,6 +859,14 @@ func (e *EventRecord) ConsumeUTF16String(offset uint16) string { return utf16.Decode(s[:len(s)/2-1]) } +// ConsumeRawUTF16String reads the byte slice for the UTF16-encoded string. +func (e *EventRecord) ConsumeRawUTF16String(offset uint16) []uint16 { + if offset > e.BufferLen { + return nil + } + return (*[1<<30 - 1]uint16)(unsafe.Pointer(e.Buffer + uintptr(offset)))[: e.BufferLen-offset : e.BufferLen-offset] +} + // ReadSID reads the security identifier from the event buffer. func (e *EventRecord) ReadSID(offset uint16, isWbemSid bool) ([]byte, uint16) { // this is a Security Token which can be null and takes 4 bytes. @@ -885,18 +895,28 @@ func (e *EventRecord) ReadSID(offset uint16, isWbemSid bool) ([]byte, uint16) { // ReadCallstack reads callstack return addresses at the given offset. func (e *EventRecord) ReadCallstack(offset uint16) []va.Address { + return e.ReadCallstackInto(offset, nil) +} + +// ReadCallstackInto reads callstack return addresses at the given offset, +// appending into v. If v has sufficient capacity, no allocation occurs. +func (e *EventRecord) ReadCallstackInto(offset uint16, v va.Callstack) []va.Address { var n uint16 m := offset - frames := (e.BufferLen - offset) / 8 - callstack := make([]va.Address, frames) + + if v != nil { + v = v[:0] + } else { + v = make([]va.Address, 0, frames) + } + for n < frames { - callstack[n] = va.Address(e.ReadUint64(m)) + v = append(v, va.Address(e.ReadUint64(m))) m += 8 n++ } - - return callstack + return v } // EventExtendedItemStackTrace64 defines a call stack on a 64-bit machine. diff --git a/pkg/util/va/address.go b/pkg/util/va/address.go index bbbf508b7..fbaadaf2f 100644 --- a/pkg/util/va/address.go +++ b/pkg/util/va/address.go @@ -18,11 +18,26 @@ package va -import "strconv" +import ( + "strconv" + "sync" +) // Address represents the memory address type Address uint64 +// Callstack is the type alias for callstack return addresses +type Callstack []Address + +const callstackDepthHint = 60 + +var callstackPool = sync.Pool{ + New: func() any { + c := make(Callstack, 0, callstackDepthHint) + return &c + }, +} + // Hex returns the hexadecimal representation of the memory address. func (a Address) String() string { return strconv.FormatUint(uint64(a), 16) } func (a Address) Uint64() uint64 { return uint64(a) } @@ -44,3 +59,15 @@ func (a Address) Dec(offset uint64) Address { // InSystemRange determines if this address is in the system address space range. // The kernel preferentially uses these two ranges to load DLLs at shared addresses. func (a Address) InSystemRange() bool { return a >= 0xfffff80000000000 && a < 0xffffffffffffffff } + +func GetCallstack() Callstack { + return (*callstackPool.Get().(*Callstack))[:0] +} + +func (c Callstack) ReleasePool() { + if cap(c) > callstackDepthHint*4 { + return // drop outlier-sized buffers rather than pooling them permanently + } + c = c[:0] + callstackPool.Put(&c) +} diff --git a/pkg/util/wildcard/wildcard.go b/pkg/util/wildcard/wildcard.go index b0f7d8f70..23f2fd664 100644 --- a/pkg/util/wildcard/wildcard.go +++ b/pkg/util/wildcard/wildcard.go @@ -12,21 +12,30 @@ package wildcard -import "unicode/utf8" +import ( + "unicode" + "unicode/utf8" +) // Match performs ASCII-first, iterative wildcard matching with UTF-8 fallback. -// It supports '*' and '?' wildcards in the pattern string. -func Match(pattern, str string) bool { - slen := len(str) - plen := len(pattern) +// It supports '*' and '?' wildcards in the pattern string. If caseSensitive +// is false, comparisons are performed case-insensitively with zero extra +// allocations. +func Match(pattern, str string, caseSensitive bool) bool { + if caseSensitive { + return matchCaseSensitive(pattern, str) + } + return matchCaseInsensitive(pattern, str) +} +func matchCaseSensitive(pattern, str string) bool { + slen, plen := len(str), len(pattern) var p, s int wildcardIdx, matchIdx := -1, 0 for s < slen { if p < plen { pb := pattern[p] - switch pb { case '?': // match exactly one character @@ -40,18 +49,14 @@ func Match(pattern, str string) bool { s += ssize } continue - case '*': - // record wildcard position - wildcardIdx = p - matchIdx = s + wildcardIdx, matchIdx = p, s p++ continue - default: // literal match if pb < utf8.RuneSelf && str[s] < utf8.RuneSelf { - if pb == str[s] { + if pb == str[s] { // direct compare, no function call, inlines trivially p++ s++ continue @@ -67,23 +72,79 @@ func Match(pattern, str string) bool { } } } - - // backtrack if there was a previous '*' if wildcardIdx != -1 { p = wildcardIdx + 1 matchIdx++ s = matchIdx continue } - - // previous '*', and mismatch return false } - - // Skip remaining stars in pattern for p < plen && pattern[p] == '*' { p++ } + return p == plen +} + +func matchCaseInsensitive(pattern, str string) bool { + slen, plen := len(str), len(pattern) + var p, s int + wildcardIdx, matchIdx := -1, 0 + for s < slen { + if p < plen { + pb := pattern[p] + switch pb { + case '?': + if str[s] < utf8.RuneSelf && pb < utf8.RuneSelf { + p++ + s++ + } else { + _, psize := utf8.DecodeRuneInString(pattern[p:]) + _, ssize := utf8.DecodeRuneInString(str[s:]) + p += psize + s += ssize + } + continue + case '*': + wildcardIdx, matchIdx = p, s + p++ + continue + default: + if pb < utf8.RuneSelf && str[s] < utf8.RuneSelf { + if pb == str[s] || foldASCII(pb) == foldASCII(str[s]) { // pb==str[s] short-circuits the common exact-match case + p++ + s++ + continue + } + } else { + pr, psize := utf8.DecodeRuneInString(pattern[p:]) + sr, ssize := utf8.DecodeRuneInString(str[s:]) + if pr == sr || unicode.ToLower(pr) == unicode.ToLower(sr) { + p += psize + s += ssize + continue + } + } + } + } + if wildcardIdx != -1 { + p = wildcardIdx + 1 + matchIdx++ + s = matchIdx + continue + } + return false + } + for p < plen && pattern[p] == '*' { + p++ + } return p == plen } + +func foldASCII(b byte) byte { + if b >= 'A' && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} diff --git a/pkg/util/wildcard/wildcard_test.go b/pkg/util/wildcard/wildcard_test.go index 19eb90f07..a993a42c0 100644 --- a/pkg/util/wildcard/wildcard_test.go +++ b/pkg/util/wildcard/wildcard_test.go @@ -20,24 +20,27 @@ import ( func TestMatch(t *testing.T) { var tests = []struct { - p string - s string - match bool + p string + s string + caseSensitive bool + match bool }{ - {"C:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true}, - {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true}, - {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\cmd.exe", false}, - {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\ActionList.xml", true}, - {"C:\\ProgramData\\*.dll", "C:\\ProgramData\\Directory\\OneMoreDirectory\\mal.dll", true}, - {"HKEY_USERS\\*\\Environment\\windir", "HKEY_USERS\\S-1-5-21-2271034452-2606270099-984871569-1001\\Environment\\windir", true}, - {"C:\\Windows\\SoftwareDistribution\\*", "C:\\Windows\\SoftwareDistribution\\SLS\\7971F918-A847-4430-9279-4A52D1EFE18D\\sls.rar", true}, - {"HKEY_USERS\\S-1-5-21-*_CLASSES\\MS-SETTINGS\\CURVER", "HKEY_USERS\\S-1-5-21-2271034452-1207270099-244871569-1021_CLASSES\\MS-SETTINGS\\CURVER", true}, - {"ntdll.dll|KernelBase.dll|advapi32.dll|*", "ntdll.dll|KernelBase.dll|advapi32.dll|pe386.dll|com.dll|clr.dll|mmc.exe", true}, + {"C:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, true}, + {"C:\\*\\LSASS?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, false}, + {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\lsass2.dmp", true, true}, + {"?:\\*\\lsass?.dmp", "C:\\Windows\\System32\\cmd.exe", true, false}, + {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\ActionList.xml", true, true}, + {"C:\\*\\ActionList.x?l", "C:\\Windows\\Setup\\LatentAcquisition\\actionList.xml", true, false}, + {"C:\\ProgramData\\*.dll", "C:\\ProgramData\\Directory\\OneMoreDirectory\\mal.dll", true, true}, + {"HKEY_USERS\\*\\Environment\\windir", "HKEY_USERS\\S-1-5-21-2271034452-2606270099-984871569-1001\\Environment\\windir", true, true}, + {"C:\\Windows\\SoftwareDistribution\\*", "C:\\Windows\\SoftwareDistribution\\SLS\\7971F918-A847-4430-9279-4A52D1EFE18D\\sls.rar", true, true}, + {"HKEY_USERS\\S-1-5-21-*_CLASSES\\MS-SETTINGS\\CURVER", "HKEY_USERS\\S-1-5-21-2271034452-1207270099-244871569-1021_CLASSES\\MS-SETTINGS\\CURVER", true, true}, + {"ntdll.dll|KernelBase.dll|advapi32.dll|*", "ntdll.dll|KernelBase.dll|advapi32.dll|pe386.dll|com.dll|clr.dll|mmc.exe", true, true}, } for _, tt := range tests { t.Run(tt.p, func(t *testing.T) { - assert.Equal(t, tt.match, Match(tt.p, tt.s)) + assert.Equal(t, tt.match, Match(tt.p, tt.s, tt.caseSensitive)) }) } } diff --git a/pkg/yara/config/config.go b/pkg/yara/config/config.go index 084adfdc0..c3385b116 100644 --- a/pkg/yara/config/config.go +++ b/pkg/yara/config/config.go @@ -21,6 +21,9 @@ package config import ( "bytes" "fmt" + "text/template" + "time" + "github.com/mitchellh/mapstructure" "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" @@ -28,9 +31,6 @@ import ( ytypes "github.com/rabbitstack/fibratus/pkg/yara/types" "github.com/spf13/pflag" "github.com/spf13/viper" - "strings" - "text/template" - "time" ) const ( @@ -143,7 +143,7 @@ func AddFlags(flags *pflag.FlagSet) { // Wildcard matching is possible. func (c Config) ShouldSkipProcess(proc string) bool { for _, p := range c.ExcludedProcesses { - if wildcard.Match(strings.ToLower(p), strings.ToLower(proc)) { + if wildcard.Match(p, proc, false) { return true } } @@ -153,7 +153,7 @@ func (c Config) ShouldSkipProcess(proc string) bool { // ShouldSkipFile determines whether the specified full file path is rejected by the scanner. func (c Config) ShouldSkipFile(file string) bool { for _, f := range c.ExcludedFiles { - if wildcard.Match(strings.ToLower(f), strings.ToLower(file)) { + if wildcard.Match(f, file, false) { return true } }