diff --git a/go.mod b/go.mod index b6d9dc10c..be2ba1d2a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/shipwright-io/cli go 1.25.6 require ( - github.com/google/go-containerregistry v0.21.2 + github.com/google/go-containerregistry v0.21.6 github.com/onsi/gomega v1.39.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/schollz/progressbar/v3 v3.19.0 @@ -33,11 +33,9 @@ require ( github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/docker/cli v29.2.1+incompatible // indirect - github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/cli v29.4.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect @@ -78,10 +76,9 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.4 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/spdystream v0.5.0 // indirect github.com/moby/term v0.5.0 // indirect @@ -102,9 +99,8 @@ require ( github.com/prometheus/statsd_exporter v0.28.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/tektoncd/pipeline v1.9.1 // indirect - github.com/vbatts/tar-split v0.12.2 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.opencensus.io v0.24.0 // indirect @@ -114,9 +110,9 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/net v0.51.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.44.0 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect @@ -130,6 +126,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gotest.tools/v3 v3.5.2 // indirect k8s.io/component-base v0.34.4 // indirect k8s.io/component-helpers v0.34.4 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect diff --git a/go.sum b/go.sum index 7393d314a..efffa6492 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudevents/sdk-go/v2 v2.16.2 h1:ZYDFrYke4FD+jM8TZTJJO6JhKHzOQl2oqpFK1D+NnQM= github.com/cloudevents/sdk-go/v2 v2.16.2/go.mod h1:laOcGImm4nVJEU+PHnUrKL56CKmRL65RlQF0kRmW/kg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= -github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -93,10 +91,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/cli v29.2.1+incompatible h1:n3Jt0QVCN65eiVBoUTZQM9mcQICCJt3akW4pKAbKdJg= -github.com/docker/cli v29.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= -github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/cli v29.4.3+incompatible h1:u+UliYm2J/rYrIh2FqHQg32neRG8GjbvNuwQRTzGspU= +github.com/docker/cli v29.4.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= @@ -235,8 +231,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.2 h1:vYaMU4nU55JJGFC9JR/s8NZcTjbE9DBBbvusTW9NeS0= -github.com/google/go-containerregistry v0.21.2/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0= +github.com/google/go-containerregistry v0.21.6 h1:T+yqQIlJXKrM98Om4DlW3GoWQAmhZuLMwoDOvVrtiUM= +github.com/google/go-containerregistry v0.21.6/go.mod h1:U7MMSBIJynke2MVQrQk19NP9k/uQsGz/h0amIFSHMbo= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -285,8 +281,8 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -308,8 +304,6 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= @@ -402,8 +396,8 @@ github.com/shipwright-io/build v0.19.0/go.mod h1:/+ZJb4UQrXF9bjNS86lXrcvDpfjcu2e github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -430,8 +424,6 @@ github.com/tektoncd/pipeline v1.9.1 h1:Js6NQleJLoo5vrS6ebg+WoHntMDY6xMS9zDvbGR2R github.com/tektoncd/pipeline v1.9.1/go.mod h1:PTlIZ4Mhr8HZDx404O7spJtafiynetTMedCsXStjtHk= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= -github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= -github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= @@ -450,16 +442,16 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -511,8 +503,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -556,8 +548,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -569,8 +561,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -613,9 +605,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= @@ -676,8 +667,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -801,8 +792,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE b/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go deleted file mode 100644 index a9e1b72ba..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ /dev/null @@ -1,756 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "os" - "path" - "runtime" - "strings" - "sync" - "sync/atomic" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/errgroup" -) - -type GzipHelperFunc func(io.Reader) (io.ReadCloser, error) - -type options struct { - chunkSize int - compressionLevel int - prioritizedFiles []string - missedPrioritizedFiles *[]string - compression Compression - ctx context.Context - minChunkSize int - gzipHelperFunc GzipHelperFunc -} - -type Option func(o *options) error - -// WithChunkSize option specifies the chunk size of eStargz blob to build. -func WithChunkSize(chunkSize int) Option { - return func(o *options) error { - o.chunkSize = chunkSize - return nil - } -} - -// WithCompressionLevel option specifies the gzip compression level. -// The default is gzip.BestCompression. -// This option will be ignored if WithCompression option is used. -// See also: https://godoc.org/compress/gzip#pkg-constants -func WithCompressionLevel(level int) Option { - return func(o *options) error { - o.compressionLevel = level - return nil - } -} - -// WithPrioritizedFiles option specifies the list of prioritized files. -// These files must be complete paths that are absolute or relative to "/" -// For example, all of "foo/bar", "/foo/bar", "./foo/bar" and "../foo/bar" -// are treated as "/foo/bar". -func WithPrioritizedFiles(files []string) Option { - return func(o *options) error { - o.prioritizedFiles = files - return nil - } -} - -// WithAllowPrioritizeNotFound makes Build continue the execution even if some -// of prioritized files specified by WithPrioritizedFiles option aren't found -// in the input tar. Instead, this records all missed file names to the passed -// slice. -func WithAllowPrioritizeNotFound(missedFiles *[]string) Option { - return func(o *options) error { - if missedFiles == nil { - return fmt.Errorf("WithAllowPrioritizeNotFound: slice must be passed") - } - o.missedPrioritizedFiles = missedFiles - return nil - } -} - -// WithCompression specifies compression algorithm to be used. -// Default is gzip. -func WithCompression(compression Compression) Option { - return func(o *options) error { - o.compression = compression - return nil - } -} - -// WithContext specifies a context that can be used for clean canceleration. -func WithContext(ctx context.Context) Option { - return func(o *options) error { - o.ctx = ctx - return nil - } -} - -// WithMinChunkSize option specifies the minimal number of bytes of data -// must be written in one gzip stream. -// By increasing this number, one gzip stream can contain multiple files -// and it hopefully leads to smaller result blob. -// NOTE: This adds a TOC property that old reader doesn't understand. -func WithMinChunkSize(minChunkSize int) Option { - return func(o *options) error { - o.minChunkSize = minChunkSize - return nil - } -} - -// WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers. -// When a gzip-compressed layer is detected, this function will be used instead of the -// Go standard library gzip decompression for better performance. -// The function should take an io.Reader as input and return an io.ReadCloser. -// If nil, the Go standard library gzip.NewReader will be used. -func WithGzipHelperFunc(gzipHelperFunc GzipHelperFunc) Option { - return func(o *options) error { - o.gzipHelperFunc = gzipHelperFunc - return nil - } -} - -// Blob is an eStargz blob. -type Blob struct { - io.ReadCloser - diffID digest.Digester - tocDigest digest.Digest - readCompleted *atomic.Bool - uncompressedSize *atomic.Int64 -} - -// DiffID returns the digest of uncompressed blob. -// It is only valid to call DiffID after Close. -func (b *Blob) DiffID() digest.Digest { - return b.diffID.Digest() -} - -// TOCDigest returns the digest of uncompressed TOC JSON. -func (b *Blob) TOCDigest() digest.Digest { - return b.tocDigest -} - -// UncompressedSize returns the size of uncompressed blob. -// UncompressedSize should only be called after the blob has been fully read. -func (b *Blob) UncompressedSize() (int64, error) { - switch { - case b.uncompressedSize == nil || b.readCompleted == nil: - return -1, fmt.Errorf("readCompleted or uncompressedSize is not initialized") - case !b.readCompleted.Load(): - return -1, fmt.Errorf("called UncompressedSize before the blob has been fully read") - default: - return b.uncompressedSize.Load(), nil - } -} - -// Build builds an eStargz blob which is an extended version of stargz, from a blob (gzip, zstd -// or plain tar) passed through the argument. If there are some prioritized files are listed in -// the option, these files are grouped as "prioritized" and can be used for runtime optimization -// (e.g. prefetch). This function builds a blob in parallel, with dividing that blob into several -// (at least the number of runtime.GOMAXPROCS(0)) sub-blobs. -func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { - var opts options - opts.compressionLevel = gzip.BestCompression // BestCompression by default - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - if opts.compression == nil { - opts.compression = newGzipCompressionWithLevel(opts.compressionLevel) - } - layerFiles := newTempFiles() - ctx := opts.ctx - if ctx == nil { - ctx = context.Background() - } - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-done: - // nop - case <-ctx.Done(): - layerFiles.CleanupAll() - } - }() - defer func() { - if rErr != nil { - if err := layerFiles.CleanupAll(); err != nil { - rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr) - } - } - if cErr := ctx.Err(); cErr != nil { - rErr = fmt.Errorf("error from context %q: %w", cErr, rErr) - } - }() - tarBlob, err := decompressBlob(tarBlob, layerFiles, opts.gzipHelperFunc) - if err != nil { - return nil, err - } - entries, err := sortEntries(tarBlob, opts.prioritizedFiles, opts.missedPrioritizedFiles) - if err != nil { - return nil, err - } - var tarParts [][]*entry - if opts.minChunkSize > 0 { - // Each entry needs to know the size of the current gzip stream so they - // cannot be processed in parallel. - tarParts = [][]*entry{entries} - } else { - tarParts = divideEntries(entries, runtime.GOMAXPROCS(0)) - } - writers := make([]*Writer, len(tarParts)) - payloads := make([]*os.File, len(tarParts)) - var mu sync.Mutex - var eg errgroup.Group - for i, parts := range tarParts { - i, parts := i, parts - // builds verifiable stargz sub-blobs - eg.Go(func() error { - esgzFile, err := layerFiles.TempFile("", "esgzdata") - if err != nil { - return err - } - sw := NewWriterWithCompressor(esgzFile, opts.compression) - sw.ChunkSize = opts.chunkSize - sw.MinChunkSize = opts.minChunkSize - if sw.needsOpenGzEntries == nil { - sw.needsOpenGzEntries = make(map[string]struct{}) - } - for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} { - sw.needsOpenGzEntries[f] = struct{}{} - } - if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { - return err - } - mu.Lock() - writers[i] = sw - payloads[i] = esgzFile - mu.Unlock() - return nil - }) - } - if err := eg.Wait(); err != nil { - rErr = err - return nil, err - } - tocAndFooter, tocDgst, err := closeWithCombine(writers...) - if err != nil { - rErr = err - return nil, err - } - var rs []io.Reader - for _, p := range payloads { - fs, err := fileSectionReader(p) - if err != nil { - return nil, err - } - rs = append(rs, fs) - } - diffID := digest.Canonical.Digester() - pr, pw := io.Pipe() - readCompleted := new(atomic.Bool) - uncompressedSize := new(atomic.Int64) - go func() { - var size int64 - var decompressFunc func(io.Reader) (io.ReadCloser, error) - if _, ok := opts.compression.(*gzipCompression); ok && opts.gzipHelperFunc != nil { - decompressFunc = opts.gzipHelperFunc - } else { - decompressFunc = opts.compression.Reader - } - decompressR, err := decompressFunc(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) - if err != nil { - pw.CloseWithError(err) - return - } - defer decompressR.Close() - if size, err = io.Copy(diffID.Hash(), decompressR); err != nil { - pw.CloseWithError(err) - return - } - uncompressedSize.Store(size) - readCompleted.Store(true) - pw.Close() - }() - return &Blob{ - ReadCloser: readCloser{ - Reader: pr, - closeFunc: layerFiles.CleanupAll, - }, - tocDigest: tocDgst, - diffID: diffID, - readCompleted: readCompleted, - uncompressedSize: uncompressedSize, - }, nil -} - -// closeWithCombine takes unclosed Writers and close them. This also returns the -// toc that combined all Writers into. -// Writers doesn't write TOC and footer to the underlying writers so they can be -// combined into a single eStargz and tocAndFooter returned by this function can -// be appended at the tail of that combined blob. -func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) { - if len(ws) == 0 { - return nil, "", fmt.Errorf("at least one writer must be passed") - } - for _, w := range ws { - if w.closed { - return nil, "", fmt.Errorf("writer must be unclosed") - } - defer func(w *Writer) { w.closed = true }(w) - if err := w.closeGz(); err != nil { - return nil, "", err - } - if err := w.bw.Flush(); err != nil { - return nil, "", err - } - } - var ( - mtoc = new(JTOC) - currentOffset int64 - ) - mtoc.Version = ws[0].toc.Version - for _, w := range ws { - for _, e := range w.toc.Entries { - // Recalculate Offset of non-empty files/chunks - if (e.Type == "reg" && e.Size > 0) || e.Type == "chunk" { - e.Offset += currentOffset - } - mtoc.Entries = append(mtoc.Entries, e) - } - if w.toc.Version > mtoc.Version { - mtoc.Version = w.toc.Version - } - currentOffset += w.cw.n - } - - return tocAndFooter(ws[0].compressor, mtoc, currentOffset) -} - -func tocAndFooter(compressor Compressor, toc *JTOC, offset int64) (io.Reader, digest.Digest, error) { - buf := new(bytes.Buffer) - tocDigest, err := compressor.WriteTOCAndFooter(buf, offset, toc, nil) - if err != nil { - return nil, "", err - } - return buf, tocDigest, nil -} - -// divideEntries divides passed entries to the parts at least the number specified by the -// argument. -func divideEntries(entries []*entry, minPartsNum int) (set [][]*entry) { - var estimatedSize int64 - for _, e := range entries { - estimatedSize += e.header.Size - } - unitSize := estimatedSize / int64(minPartsNum) - var ( - nextEnd = unitSize - offset int64 - ) - set = append(set, []*entry{}) - for _, e := range entries { - set[len(set)-1] = append(set[len(set)-1], e) - offset += e.header.Size - if offset > nextEnd { - set = append(set, []*entry{}) - nextEnd += unitSize - } - } - return -} - -var errNotFound = errors.New("not found") - -// sortEntries reads the specified tar blob and returns a list of tar entries. -// If some of prioritized files are specified, the list starts from these -// files with keeping the order specified by the argument. -func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]string) ([]*entry, error) { - - // Import tar file. - intar, err := importTar(in) - if err != nil { - return nil, fmt.Errorf("failed to sort: %w", err) - } - - // Sort the tar file respecting to the prioritized files list. - sorted := &tarFile{} - picked := make(map[string]struct{}) - for _, l := range prioritized { - if err := moveRec(l, intar, sorted, picked); err != nil { - if errors.Is(err, errNotFound) && missedPrioritized != nil { - *missedPrioritized = append(*missedPrioritized, l) - continue // allow not found - } - return nil, fmt.Errorf("failed to sort tar entries: %w", err) - } - } - if len(prioritized) == 0 { - sorted.add(&entry{ - header: &tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } else { - sorted.add(&entry{ - header: &tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } - - // Dump prioritized entries followed by the rest entries while skipping picked ones. - return append(sorted.dump(nil), intar.dump(picked)...), nil -} - -// readerFromEntries returns a reader of tar archive that contains entries passed -// through the arguments. -func readerFromEntries(entries ...*entry) io.Reader { - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - defer tw.Close() - for _, entry := range entries { - if err := tw.WriteHeader(entry.header); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar header: %v", err)) - return - } - if _, err := io.Copy(tw, entry.payload); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar payload: %v", err)) - return - } - } - pw.Close() - }() - return pr -} - -func importTar(in io.ReaderAt) (*tarFile, error) { - tf := &tarFile{} - pw, err := newCountReadSeeker(in) - if err != nil { - return nil, fmt.Errorf("failed to make position watcher: %w", err) - } - tr := tar.NewReader(pw) - - // Walk through all nodes. - for { - // Fetch and parse next header. - h, err := tr.Next() - if err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("failed to parse tar file, %w", err) - } - switch cleanEntryName(h.Name) { - case PrefetchLandmark, NoPrefetchLandmark: - // Ignore existing landmark - continue - } - - // Add entry. If it already exists, replace it. - if _, ok := tf.get(h.Name); ok { - tf.remove(h.Name) - } - tf.add(&entry{ - header: h, - payload: io.NewSectionReader(in, pw.currentPos(), h.Size), - }) - } - - return tf, nil -} - -func moveRec(name string, in *tarFile, out *tarFile, picked map[string]struct{}) error { - name = cleanEntryName(name) - if name == "" { // root directory. stop recursion. - if e, ok := in.get(name); ok { - // entry of the root directory exists. we should move it as well. - // this case will occur if tar entries are prefixed with "./", "/", etc. - if _, done := picked[name]; !done { - out.add(e) - picked[name] = struct{}{} - } - } - return nil - } - - _, okIn := in.get(name) - _, okOut := out.get(name) - _, okPicked := picked[name] - if !okIn && !okOut && !okPicked { - return fmt.Errorf("file: %q: %w", name, errNotFound) - } - - parent, _ := path.Split(strings.TrimSuffix(name, "/")) - if err := moveRec(parent, in, out, picked); err != nil { - return err - } - if e, ok := in.get(name); ok && e.header.Typeflag == tar.TypeLink { - if err := moveRec(e.header.Linkname, in, out, picked); err != nil { - return err - } - } - if _, done := picked[name]; done { - return nil - } - if e, ok := in.get(name); ok { - out.add(e) - picked[name] = struct{}{} - } - return nil -} - -type entry struct { - header *tar.Header - payload io.ReadSeeker -} - -type tarFile struct { - index map[string]*entry - stream []*entry -} - -func (f *tarFile) add(e *entry) { - if f.index == nil { - f.index = make(map[string]*entry) - } - f.index[cleanEntryName(e.header.Name)] = e - f.stream = append(f.stream, e) -} - -func (f *tarFile) remove(name string) { - name = cleanEntryName(name) - if f.index != nil { - delete(f.index, name) - } - var filtered []*entry - for _, e := range f.stream { - if cleanEntryName(e.header.Name) == name { - continue - } - filtered = append(filtered, e) - } - f.stream = filtered -} - -func (f *tarFile) get(name string) (e *entry, ok bool) { - if f.index == nil { - return nil, false - } - e, ok = f.index[cleanEntryName(name)] - return -} - -func (f *tarFile) dump(skip map[string]struct{}) []*entry { - if len(skip) == 0 { - return f.stream - } - var out []*entry - for _, e := range f.stream { - if _, ok := skip[cleanEntryName(e.header.Name)]; ok { - continue - } - out = append(out, e) - } - return out -} - -type readCloser struct { - io.Reader - closeFunc func() error -} - -func (rc readCloser) Close() error { - return rc.closeFunc() -} - -func fileSectionReader(file *os.File) (*io.SectionReader, error) { - info, err := file.Stat() - if err != nil { - return nil, err - } - return io.NewSectionReader(file, 0, info.Size()), nil -} - -func newTempFiles() *tempFiles { - return &tempFiles{} -} - -type tempFiles struct { - files []*os.File - filesMu sync.Mutex - cleanupOnce sync.Once -} - -func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) { - f, err := os.CreateTemp(dir, pattern) - if err != nil { - return nil, err - } - tf.filesMu.Lock() - tf.files = append(tf.files, f) - tf.filesMu.Unlock() - return f, nil -} - -func (tf *tempFiles) CleanupAll() (err error) { - tf.cleanupOnce.Do(func() { - err = tf.cleanupAll() - }) - return -} - -func (tf *tempFiles) cleanupAll() error { - tf.filesMu.Lock() - defer tf.filesMu.Unlock() - var allErr []error - for _, f := range tf.files { - if err := f.Close(); err != nil { - allErr = append(allErr, err) - } - if err := os.Remove(f.Name()); err != nil { - allErr = append(allErr, err) - } - } - tf.files = nil - return errorutil.Aggregate(allErr) -} - -func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) { - pos := int64(0) - return &countReadSeeker{r: r, cPos: &pos}, nil -} - -type countReadSeeker struct { - r io.ReaderAt - cPos *int64 - - mu sync.Mutex -} - -func (cr *countReadSeeker) Read(p []byte) (int, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - n, err := cr.r.ReadAt(p, *cr.cPos) - if err == nil { - *cr.cPos += int64(n) - } - return n, err -} - -func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - switch whence { - default: - return 0, fmt.Errorf("unknown whence: %v", whence) - case io.SeekStart: - case io.SeekCurrent: - offset += *cr.cPos - case io.SeekEnd: - return 0, fmt.Errorf("unsupported whence: %v", whence) - } - - if offset < 0 { - return 0, fmt.Errorf("invalid offset") - } - *cr.cPos = offset - return offset, nil -} - -func (cr *countReadSeeker) currentPos() int64 { - cr.mu.Lock() - defer cr.mu.Unlock() - - return *cr.cPos -} - -func decompressBlob(org *io.SectionReader, tmp *tempFiles, gzipHelperFunc GzipHelperFunc) (*io.SectionReader, error) { - if org.Size() < 4 { - return org, nil - } - src := make([]byte, 4) - if _, err := org.Read(src); err != nil && err != io.EOF { - return nil, err - } - var dR io.Reader - if bytes.Equal([]byte{0x1F, 0x8B, 0x08}, src[:3]) { - // gzip - var dgR io.ReadCloser - var err error - if gzipHelperFunc != nil { - dgR, err = gzipHelperFunc(io.NewSectionReader(org, 0, org.Size())) - } else { - dgR, err = gzip.NewReader(io.NewSectionReader(org, 0, org.Size())) - } - if err != nil { - return nil, err - } - defer dgR.Close() - dR = io.Reader(dgR) - } else if bytes.Equal([]byte{0x28, 0xb5, 0x2f, 0xfd}, src[:4]) { - // zstd - dzR, err := zstd.NewReader(io.NewSectionReader(org, 0, org.Size())) - if err != nil { - return nil, err - } - defer dzR.Close() - dR = io.Reader(dzR) - } else { - // uncompressed - return io.NewSectionReader(org, 0, org.Size()), nil - } - b, err := tmp.TempFile("", "uncompresseddata") - if err != nil { - return nil, err - } - if _, err := io.Copy(b, dR); err != nil { - return nil, err - } - return fileSectionReader(b) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go deleted file mode 100644 index 6de78b02d..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errorutil - -import ( - "errors" - "fmt" - "strings" -) - -// Aggregate combines a list of errors into a single new error. -func Aggregate(errs []error) error { - switch len(errs) { - case 0: - return nil - case 1: - return errs[0] - default: - points := make([]string, len(errs)+1) - points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) - for i, err := range errs { - points[i+1] = fmt.Sprintf("* %s", err) - } - return errors.New(strings.Join(points, "\n\t")) - } -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go deleted file mode 100644 index 693730420..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ /dev/null @@ -1,1232 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "bufio" - "bytes" - "compress/gzip" - "crypto/sha256" - "errors" - "fmt" - "hash" - "io" - "os" - "path" - "sort" - "strings" - "sync" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - digest "github.com/opencontainers/go-digest" - "github.com/vbatts/tar-split/archive/tar" -) - -// A Reader permits random access reads from a stargz file. -type Reader struct { - sr *io.SectionReader - toc *JTOC - tocDigest digest.Digest - - // m stores all non-chunk entries, keyed by name. - m map[string]*TOCEntry - - // chunks stores all TOCEntry values for regular files that - // are split up. For a file with a single chunk, it's only - // stored in m. - chunks map[string][]*TOCEntry - - decompressor Decompressor -} - -type openOpts struct { - tocOffset int64 - decompressors []Decompressor - telemetry *Telemetry -} - -// OpenOption is an option used during opening the layer -type OpenOption func(o *openOpts) error - -// WithTOCOffset option specifies the offset of TOC -func WithTOCOffset(tocOffset int64) OpenOption { - return func(o *openOpts) error { - o.tocOffset = tocOffset - return nil - } -} - -// WithDecompressors option specifies decompressors to use. -// Default is gzip-based decompressor. -func WithDecompressors(decompressors ...Decompressor) OpenOption { - return func(o *openOpts) error { - o.decompressors = decompressors - return nil - } -} - -// WithTelemetry option specifies the telemetry hooks -func WithTelemetry(telemetry *Telemetry) OpenOption { - return func(o *openOpts) error { - o.telemetry = telemetry - return nil - } -} - -// MeasureLatencyHook is a func which takes start time and records the diff -type MeasureLatencyHook func(time.Time) - -// Telemetry is a struct which defines telemetry hooks. By implementing these hooks you should be able to record -// the latency metrics of the respective steps of estargz open operation. To be used with estargz.OpenWithTelemetry(...) -type Telemetry struct { - GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds) - GetTocLatency MeasureLatencyHook // measure time to GET TOC JSON (in milliseconds) - DeserializeTocLatency MeasureLatencyHook // measure time to deserialize TOC JSON (in milliseconds) -} - -// Open opens a stargz file for reading. -// The behavior is configurable using options. -// -// Note that each entry name is normalized as the path that is relative to root. -func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { - var opts openOpts - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - - gzipCompressors := []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} - decompressors := append(gzipCompressors, opts.decompressors...) - - // Determine the size to fetch. Try to fetch as many bytes as possible. - fetchSize := maxFooterSize(sr.Size(), decompressors...) - if maybeTocOffset := opts.tocOffset; maybeTocOffset > fetchSize { - if maybeTocOffset > sr.Size() { - return nil, fmt.Errorf("blob size %d is smaller than the toc offset", sr.Size()) - } - fetchSize = sr.Size() - maybeTocOffset - } - - start := time.Now() // before getting layer footer - footer := make([]byte, fetchSize) - if _, err := sr.ReadAt(footer, sr.Size()-fetchSize); err != nil { - return nil, fmt.Errorf("error reading footer: %v", err) - } - if opts.telemetry != nil && opts.telemetry.GetFooterLatency != nil { - opts.telemetry.GetFooterLatency(start) - } - - var allErr []error - var found bool - var r *Reader - for _, d := range decompressors { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - maybeTocBytes := footer[:fOffset] - _, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) - if err != nil { - allErr = append(allErr, err) - continue - } - if tocOffset >= 0 && tocSize <= 0 { - tocSize = sr.Size() - tocOffset - fSize - } - if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) { - maybeTocBytes = maybeTocBytes[:tocSize] - } - r, err = parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes, opts) - if err == nil { - found = true - break - } - allErr = append(allErr, err) - } - if !found { - return nil, errorutil.Aggregate(allErr) - } - if err := r.initFields(); err != nil { - return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) - } - return r, nil -} - -// OpenFooter extracts and parses footer from the given blob. -// only supports gzip-based eStargz. -func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr error) { - if sr.Size() < FooterSize && sr.Size() < legacyFooterSize { - return 0, 0, fmt.Errorf("blob size %d is smaller than the footer size", sr.Size()) - } - var footer [FooterSize]byte - if _, err := sr.ReadAt(footer[:], sr.Size()-FooterSize); err != nil { - return 0, 0, fmt.Errorf("error reading footer: %v", err) - } - var allErr []error - for _, d := range []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - _, tocOffset, _, err := d.ParseFooter(footer[fOffset:]) - if err == nil { - return tocOffset, fSize, err - } - allErr = append(allErr, err) - } - return 0, 0, errorutil.Aggregate(allErr) -} - -// initFields populates the Reader from r.toc after decoding it from -// JSON. -// -// Unexported fields are populated and TOCEntry fields that were -// implicit in the JSON are populated. -func (r *Reader) initFields() error { - r.m = make(map[string]*TOCEntry, len(r.toc.Entries)) - r.chunks = make(map[string][]*TOCEntry) - var lastPath string - uname := map[int]string{} - gname := map[int]string{} - var lastRegEnt *TOCEntry - var chunkTopIndex int - for i, ent := range r.toc.Entries { - ent.Name = cleanEntryName(ent.Name) - switch ent.Type { - case "reg", "chunk": - if ent.Offset != r.toc.Entries[chunkTopIndex].Offset { - chunkTopIndex = i - } - ent.chunkTopIndex = chunkTopIndex - } - if ent.Type == "reg" { - lastRegEnt = ent - } - if ent.Type == "chunk" { - ent.Name = lastPath - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - if ent.ChunkSize == 0 && lastRegEnt != nil { - ent.ChunkSize = lastRegEnt.Size - ent.ChunkOffset - } - } else { - lastPath = ent.Name - - if ent.Uname != "" { - uname[ent.UID] = ent.Uname - } else { - ent.Uname = uname[ent.UID] - } - if ent.Gname != "" { - gname[ent.GID] = ent.Gname - } else { - ent.Gname = gname[ent.GID] - } - - ent.modTime, _ = time.Parse(time.RFC3339, ent.ModTime3339) - - if ent.Type == "dir" { - ent.NumLink++ // Parent dir links to this directory - } - r.m[ent.Name] = ent - } - if ent.Type == "reg" && ent.ChunkSize > 0 && ent.ChunkSize < ent.Size { - r.chunks[ent.Name] = make([]*TOCEntry, 0, ent.Size/ent.ChunkSize+1) - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - } - if ent.ChunkSize == 0 && ent.Size != 0 { - ent.ChunkSize = ent.Size - } - } - - // Populate children, add implicit directories: - for _, ent := range r.toc.Entries { - if ent.Type == "chunk" { - continue - } - // add "foo/": - // add "foo" child to "" (creating "" if necessary) - // - // add "foo/bar/": - // add "bar" child to "foo" (creating "foo" if necessary) - // - // add "foo/bar.txt": - // add "bar.txt" child to "foo" (creating "foo" if necessary) - // - // add "a/b/c/d/e/f.txt": - // create "a/b/c/d/e" node - // add "f.txt" child to "e" - - name := ent.Name - pdirName := parentDir(name) - if name == pdirName { - // This entry and its parent are the same. - // Ignore this for avoiding infinite loop of the reference. - // The example case where this can occur is when tar contains the root - // directory itself (e.g. "./", "/"). - continue - } - pdir := r.getOrCreateDir(pdirName) - ent.NumLink++ // at least one name(ent.Name) references this entry. - if ent.Type == "hardlink" { - org, err := r.getSource(ent) - if err != nil { - return err - } - org.NumLink++ // original entry is referenced by this ent.Name. - ent = org - } - pdir.addChild(path.Base(name), ent) - } - - lastOffset := r.sr.Size() - for i := len(r.toc.Entries) - 1; i >= 0; i-- { - e := r.toc.Entries[i] - if e.isDataType() { - e.nextOffset = lastOffset - } - if e.Offset != 0 && e.InnerOffset == 0 { - lastOffset = e.Offset - } - } - - if len(r.m) == 0 { - r.m[""] = &TOCEntry{ - Name: "", - Type: "dir", - Mode: 0755, - NumLink: 1, - } - } - - return nil -} - -func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) { - if ent.Type == "hardlink" { - org, ok := r.m[cleanEntryName(ent.LinkName)] - if !ok { - return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName) - } - ent, err = r.getSource(org) - if err != nil { - return nil, err - } - } - return ent, nil -} - -func parentDir(p string) string { - dir, _ := path.Split(p) - return strings.TrimSuffix(dir, "/") -} - -func (r *Reader) getOrCreateDir(d string) *TOCEntry { - e, ok := r.m[d] - if !ok { - e = &TOCEntry{ - Name: d, - Type: "dir", - Mode: 0755, - NumLink: 2, // The directory itself(.) and the parent link to this directory. - } - r.m[d] = e - if d != "" { - pdir := r.getOrCreateDir(parentDir(d)) - pdir.addChild(path.Base(d), e) - } - } - return e -} - -func (r *Reader) TOCDigest() digest.Digest { - return r.tocDigest -} - -// VerifyTOC checks that the TOC JSON in the passed blob matches the -// passed digests and that the TOC JSON contains digests for all chunks -// contained in the blob. If the verification succceeds, this function -// returns TOCEntryVerifier which holds all chunk digests in the stargz blob. -func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) { - // Verify the digest of TOC JSON - if r.tocDigest != tocDigest { - return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest) - } - return r.Verifiers() -} - -// Verifiers returns TOCEntryVerifier of this chunk. Use VerifyTOC instead in most cases -// because this doesn't verify TOC. -func (r *Reader) Verifiers() (TOCEntryVerifier, error) { - chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest - regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest - var chunkDigestMapIncomplete bool - var regDigestMapIncomplete bool - var containsChunk bool - for _, e := range r.toc.Entries { - if e.Type != "reg" && e.Type != "chunk" { - continue - } - - // offset must be unique in stargz blob - _, dOK := chunkDigestMap[e.Offset] - _, rOK := regDigestMap[e.Offset] - if dOK || rOK { - return nil, fmt.Errorf("offset %d found twice", e.Offset) - } - - if e.Type == "reg" { - if e.Size == 0 { - continue // ignores empty file - } - - // record the digest of regular file payload - if e.Digest != "" { - d, err := digest.Parse(e.Digest) - if err != nil { - return nil, fmt.Errorf("failed to parse regular file digest %q: %w", e.Digest, err) - } - regDigestMap[e.Offset] = d - } else { - regDigestMapIncomplete = true - } - } else { - containsChunk = true // this layer contains "chunk" entries. - } - - // "reg" also can contain ChunkDigest (e.g. when "reg" is the first entry of - // chunked file) - if e.ChunkDigest != "" { - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, fmt.Errorf("failed to parse chunk digest %q: %w", e.ChunkDigest, err) - } - chunkDigestMap[e.Offset] = d - } else { - chunkDigestMapIncomplete = true - } - } - - if chunkDigestMapIncomplete { - // Though some chunk digests are not found, if this layer doesn't contain - // "chunk"s and all digest of "reg" files are recorded, we can use them instead. - if !containsChunk && !regDigestMapIncomplete { - return &verifier{digestMap: regDigestMap}, nil - } - return nil, fmt.Errorf("some ChunkDigest not found in TOC JSON") - } - - return &verifier{digestMap: chunkDigestMap}, nil -} - -// verifier is an implementation of TOCEntryVerifier which holds verifiers keyed by -// offset of the chunk. -type verifier struct { - digestMap map[int64]digest.Digest - digestMapMu sync.Mutex -} - -// Verifier returns a content verifier specified by TOCEntry. -func (v *verifier) Verifier(ce *TOCEntry) (digest.Verifier, error) { - v.digestMapMu.Lock() - defer v.digestMapMu.Unlock() - d, ok := v.digestMap[ce.Offset] - if !ok { - return nil, fmt.Errorf("verifier for offset=%d,size=%d hasn't been registered", - ce.Offset, ce.ChunkSize) - } - return d.Verifier(), nil -} - -// ChunkEntryForOffset returns the TOCEntry containing the byte of the -// named file at the given offset within the file. -// Name must be absolute path or one that is relative to root. -func (r *Reader) ChunkEntryForOffset(name string, offset int64) (e *TOCEntry, ok bool) { - name = cleanEntryName(name) - e, ok = r.Lookup(name) - if !ok || !e.isDataType() { - return nil, false - } - ents := r.chunks[name] - if len(ents) < 2 { - if offset >= e.ChunkSize { - return nil, false - } - return e, true - } - i := sort.Search(len(ents), func(i int) bool { - e := ents[i] - return e.ChunkOffset >= offset || (offset > e.ChunkOffset && offset < e.ChunkOffset+e.ChunkSize) - }) - if i == len(ents) { - return nil, false - } - return ents[i], true -} - -// Lookup returns the Table of Contents entry for the given path. -// -// To get the root directory, use the empty string. -// Path must be absolute path or one that is relative to root. -func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) { - path = cleanEntryName(path) - if r == nil { - return - } - e, ok = r.m[path] - if ok && e.Type == "hardlink" { - var err error - e, err = r.getSource(e) - if err != nil { - return nil, false - } - } - return -} - -// OpenFile returns the reader of the specified file payload. -// -// Name must be absolute path or one that is relative to root. -func (r *Reader) OpenFile(name string) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) newFileReader(name string) (*fileReader, error) { - name = cleanEntryName(name) - ent, ok := r.Lookup(name) - if !ok { - // TODO: come up with some error plan. This is lazy: - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: os.ErrNotExist, - } - } - if ent.Type != "reg" { - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: errors.New("not a regular file"), - } - } - return &fileReader{ - r: r, - size: ent.Size, - ents: r.getChunks(ent), - }, nil -} - -func (r *Reader) OpenFileWithPreReader(name string, preRead func(*TOCEntry, io.Reader) error) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - fr.preRead = preRead - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) getChunks(ent *TOCEntry) []*TOCEntry { - if ents, ok := r.chunks[ent.Name]; ok { - return ents - } - return []*TOCEntry{ent} -} - -type fileReader struct { - r *Reader - size int64 - ents []*TOCEntry // 1 or more reg/chunk entries - preRead func(*TOCEntry, io.Reader) error -} - -func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { - if off >= fr.size { - return 0, io.EOF - } - if off < 0 { - return 0, errors.New("invalid offset") - } - var i int - if len(fr.ents) > 1 { - i = sort.Search(len(fr.ents), func(i int) bool { - return fr.ents[i].ChunkOffset >= off - }) - if i == len(fr.ents) { - i = len(fr.ents) - 1 - } - } - ent := fr.ents[i] - if ent.ChunkOffset > off { - if i == 0 { - return 0, errors.New("internal error; first chunk offset is non-zero") - } - ent = fr.ents[i-1] - } - - // If ent is a chunk of a large file, adjust the ReadAt - // offset by the chunk's offset. - off -= ent.ChunkOffset - - finalEnt := fr.ents[len(fr.ents)-1] - compressedOff := ent.Offset - // compressedBytesRemain is the number of compressed bytes in this - // file remaining, over 1+ chunks. - compressedBytesRemain := finalEnt.NextOffset() - compressedOff - - sr := io.NewSectionReader(fr.r.sr, compressedOff, compressedBytesRemain) - - const maxRead = 2 << 20 - var bufSize = maxRead - if compressedBytesRemain < maxRead { - bufSize = int(compressedBytesRemain) - } - - br := bufio.NewReaderSize(sr, bufSize) - if _, err := br.Peek(bufSize); err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.peek: %v", err) - } - - dr, err := fr.r.decompressor.Reader(br) - if err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err) - } - defer dr.Close() - - if fr.preRead == nil { - if n, err := io.CopyN(io.Discard, dr, ent.InnerOffset+off); n != ent.InnerOffset+off || err != nil { - return 0, fmt.Errorf("discard of %d bytes != %v, %v", ent.InnerOffset+off, n, err) - } - return io.ReadFull(dr, p) - } - - var retN int - var retErr error - var found bool - var nr int64 - for _, e := range fr.r.toc.Entries[ent.chunkTopIndex:] { - if !e.isDataType() { - continue - } - if e.Offset != fr.r.toc.Entries[ent.chunkTopIndex].Offset { - break - } - if in, err := io.CopyN(io.Discard, dr, e.InnerOffset-nr); err != nil || in != e.InnerOffset-nr { - return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.InnerOffset-nr, in, err) - } - nr = e.InnerOffset - if e == ent { - found = true - if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil { - return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err) - } - retN, retErr = io.ReadFull(dr, p) - nr += off + int64(retN) - continue - } - cr := &countReader{r: io.LimitReader(dr, e.ChunkSize)} - if err := fr.preRead(e, cr); err != nil { - return 0, fmt.Errorf("failed to pre read: %w", err) - } - nr += cr.n - } - if !found { - return 0, fmt.Errorf("fileReader.ReadAt: target entry not found") - } - return retN, retErr -} - -// A Writer writes stargz files. -// -// Use NewWriter to create a new Writer. -type Writer struct { - bw *bufio.Writer - cw *countWriter - toc *JTOC - diffHash hash.Hash // SHA-256 of uncompressed tar - - closed bool - gz io.WriteCloser - lastUsername map[int]string - lastGroupname map[int]string - compressor Compressor - - uncompressedCounter *countWriteFlusher - - // ChunkSize optionally controls the maximum number of bytes - // of data of a regular file that can be written in one gzip - // stream before a new gzip stream is started. - // Zero means to use a default, currently 4 MiB. - ChunkSize int - - // MinChunkSize optionally controls the minimum number of bytes - // of data must be written in one gzip stream before a new gzip - // NOTE: This adds a TOC property that stargz snapshotter < v0.13.0 doesn't understand. - MinChunkSize int - - needsOpenGzEntries map[string]struct{} -} - -// currentCompressionWriter writes to the current w.gz field, which can -// change throughout writing a tar entry. -// -// Additionally, it updates w's SHA-256 of the uncompressed bytes -// of the tar file. -type currentCompressionWriter struct{ w *Writer } - -func (ccw currentCompressionWriter) Write(p []byte) (int, error) { - ccw.w.diffHash.Write(p) - if ccw.w.gz == nil { - if err := ccw.w.condOpenGz(); err != nil { - return 0, err - } - } - return ccw.w.gz.Write(p) -} - -func (w *Writer) chunkSize() int { - if w.ChunkSize <= 0 { - return 4 << 20 - } - return w.ChunkSize -} - -// Unpack decompresses the given estargz blob and returns a ReadCloser of the tar blob. -// TOC JSON and footer are removed. -func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) { - footerSize := c.FooterSize() - if sr.Size() < footerSize { - return nil, fmt.Errorf("blob is too small; %d < %d", sr.Size(), footerSize) - } - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - return nil, err - } - blobPayloadSize, _, _, err := c.ParseFooter(footer) - if err != nil { - return nil, fmt.Errorf("failed to parse footer: %w", err) - } - if blobPayloadSize < 0 { - blobPayloadSize = sr.Size() - } - return c.Reader(io.LimitReader(sr, blobPayloadSize)) -} - -// NewWriter returns a new stargz writer (gzip-based) writing to w. -// -// The writer must be closed to write its trailing table of contents. -func NewWriter(w io.Writer) *Writer { - return NewWriterLevel(w, gzip.BestCompression) -} - -// NewWriterLevel returns a new stargz writer (gzip-based) writing to w. -// The compression level is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { - return NewWriterWithCompressor(w, NewGzipCompressorWithLevel(compressionLevel)) -} - -// NewWriterWithCompressor returns a new stargz writer writing to w. -// The compression method is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer { - bw := bufio.NewWriter(w) - cw := &countWriter{w: bw} - return &Writer{ - bw: bw, - cw: cw, - toc: &JTOC{Version: 1}, - diffHash: sha256.New(), - compressor: c, - uncompressedCounter: &countWriteFlusher{}, - } -} - -// Close writes the stargz's table of contents and flushes all the -// buffers, returning any error. -func (w *Writer) Close() (digest.Digest, error) { - if w.closed { - return "", nil - } - defer func() { w.closed = true }() - - if err := w.closeGz(); err != nil { - return "", err - } - - // Write the TOC index and footer. - tocDigest, err := w.compressor.WriteTOCAndFooter(w.cw, w.cw.n, w.toc, w.diffHash) - if err != nil { - return "", err - } - if err := w.bw.Flush(); err != nil { - return "", err - } - - return tocDigest, nil -} - -func (w *Writer) closeGz() error { - if w.closed { - return errors.New("write on closed Writer") - } - if w.gz != nil { - if err := w.gz.Close(); err != nil { - return err - } - w.gz = nil - } - return nil -} - -func (w *Writer) flushGz() error { - if w.closed { - return errors.New("flush on closed Writer") - } - if w.gz != nil { - if f, ok := w.gz.(interface { - Flush() error - }); ok { - return f.Flush() - } - } - return nil -} - -// nameIfChanged returns name, unless it was the already the value of (*mp)[id], -// in which case it returns the empty string. -func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string { - if name == "" { - return "" - } - if *mp == nil { - *mp = make(map[int]string) - } - if (*mp)[id] == name { - return "" - } - (*mp)[id] = name - return name -} - -func (w *Writer) condOpenGz() (err error) { - if w.gz == nil { - w.gz, err = w.compressor.Writer(w.cw) - if w.gz != nil { - w.gz = w.uncompressedCounter.register(w.gz) - } - } - return -} - -// AppendTar reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -func (w *Writer) AppendTar(r io.Reader) error { - return w.appendTar(r, false) -} - -// AppendTarLossLess reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -// -// The difference of this func with AppendTar is that this writes -// the input tar stream into w without any modification (e.g. to header bytes). -// -// Note that if the input tar stream already contains TOC JSON, this returns -// error because w cannot overwrite the TOC JSON to the one generated by w without -// lossy modification. To avoid this error, if the input stream is known to be stargz/estargz, -// you shoud decompress it and remove TOC JSON in advance. -func (w *Writer) AppendTarLossLess(r io.Reader) error { - return w.appendTar(r, true) -} - -func (w *Writer) appendTar(r io.Reader, lossless bool) error { - var src io.Reader - br := bufio.NewReader(r) - if isGzip(br) { - zr, _ := gzip.NewReader(br) - src = zr - } else { - src = io.Reader(br) - } - dst := currentCompressionWriter{w} - var tw *tar.Writer - if !lossless { - tw = tar.NewWriter(dst) // use tar writer only when this isn't lossless mode. - } - tr := tar.NewReader(src) - if lossless { - tr.RawAccounting = true - } - prevOffset := w.cw.n - var prevOffsetUncompressed int64 - for { - h, err := tr.Next() - if err == io.EOF { - if lossless { - if remain := tr.RawBytes(); len(remain) > 0 { - // Collect the remaining null bytes. - // https://github.com/vbatts/tar-split/blob/80a436fd6164c557b131f7c59ed69bd81af69761/concept/main.go#L49-L53 - if _, err := dst.Write(remain); err != nil { - return err - } - } - } - break - } - if err != nil { - return fmt.Errorf("error reading from source tar: tar.Reader.Next: %v", err) - } - if cleanEntryName(h.Name) == TOCTarName { - // It is possible for a layer to be "stargzified" twice during the - // distribution lifecycle. So we reserve "TOCTarName" here to avoid - // duplicated entries in the resulting layer. - if lossless { - // We cannot handle this in lossless way. - return fmt.Errorf("existing TOC JSON is not allowed; decompress layer before append") - } - continue - } - - xattrs := make(map[string][]byte) - const xattrPAXRecordsPrefix = "SCHILY.xattr." - if h.PAXRecords != nil { - for k, v := range h.PAXRecords { - if strings.HasPrefix(k, xattrPAXRecordsPrefix) { - xattrs[k[len(xattrPAXRecordsPrefix):]] = []byte(v) - } - } - } - ent := &TOCEntry{ - Name: h.Name, - Mode: h.Mode, - UID: h.Uid, - GID: h.Gid, - Uname: w.nameIfChanged(&w.lastUsername, h.Uid, h.Uname), - Gname: w.nameIfChanged(&w.lastGroupname, h.Gid, h.Gname), - ModTime3339: formatModtime(h.ModTime), - Xattrs: xattrs, - } - if err := w.condOpenGz(); err != nil { - return err - } - if tw != nil { - if err := tw.WriteHeader(h); err != nil { - return err - } - } else { - if _, err := dst.Write(tr.RawBytes()); err != nil { - return err - } - } - switch h.Typeflag { - case tar.TypeLink: - ent.Type = "hardlink" - ent.LinkName = h.Linkname - case tar.TypeSymlink: - ent.Type = "symlink" - ent.LinkName = h.Linkname - case tar.TypeDir: - ent.Type = "dir" - case tar.TypeReg: - ent.Type = "reg" - ent.Size = h.Size - case tar.TypeChar: - ent.Type = "char" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeBlock: - ent.Type = "block" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeFifo: - ent.Type = "fifo" - default: - return fmt.Errorf("unsupported input tar entry %q", h.Typeflag) - } - - // We need to keep a reference to the TOC entry for regular files, so that we - // can fill the digest later. - var regFileEntry *TOCEntry - var payloadDigest digest.Digester - if h.Typeflag == tar.TypeReg { - regFileEntry = ent - payloadDigest = digest.Canonical.Digester() - } - - if h.Typeflag == tar.TypeReg && ent.Size > 0 { - var written int64 - totalSize := ent.Size // save it before we destroy ent - tee := io.TeeReader(tr, payloadDigest.Hash()) - for written < totalSize { - chunkSize := int64(w.chunkSize()) - remain := totalSize - written - if remain < chunkSize { - chunkSize = remain - } else { - ent.ChunkSize = chunkSize - } - - // We flush the underlying compression writer here to correctly calculate "w.cw.n". - if err := w.flushGz(); err != nil { - return err - } - if w.needsOpenGz(ent) || w.cw.n-prevOffset >= int64(w.MinChunkSize) { - if err := w.closeGz(); err != nil { - return err - } - ent.Offset = w.cw.n - prevOffset = ent.Offset - prevOffsetUncompressed = w.uncompressedCounter.n - } else { - ent.Offset = prevOffset - ent.InnerOffset = w.uncompressedCounter.n - prevOffsetUncompressed - } - - ent.ChunkOffset = written - chunkDigest := digest.Canonical.Digester() - - if err := w.condOpenGz(); err != nil { - return err - } - - teeChunk := io.TeeReader(tee, chunkDigest.Hash()) - var out io.Writer - if tw != nil { - out = tw - } else { - out = dst - } - if _, err := io.CopyN(out, teeChunk, chunkSize); err != nil { - return fmt.Errorf("error copying %q: %v", h.Name, err) - } - ent.ChunkDigest = chunkDigest.Digest().String() - w.toc.Entries = append(w.toc.Entries, ent) - written += chunkSize - ent = &TOCEntry{ - Name: h.Name, - Type: "chunk", - } - } - } else { - w.toc.Entries = append(w.toc.Entries, ent) - } - if payloadDigest != nil { - regFileEntry.Digest = payloadDigest.Digest().String() - } - if tw != nil { - if err := tw.Flush(); err != nil { - return err - } - } - } - remainDest := io.Discard - if lossless { - remainDest = dst // Preserve the remaining bytes in lossless mode - } - _, err := io.Copy(remainDest, src) - return err -} - -func (w *Writer) needsOpenGz(ent *TOCEntry) bool { - if ent.Type != "reg" { - return false - } - if w.needsOpenGzEntries == nil { - return false - } - _, ok := w.needsOpenGzEntries[ent.Name] - return ok -} - -// DiffID returns the SHA-256 of the uncompressed tar bytes. -// It is only valid to call DiffID after Close. -func (w *Writer) DiffID() string { - return fmt.Sprintf("sha256:%x", w.diffHash.Sum(nil)) -} - -func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) { - for _, d := range decompressors { - if s := d.FooterSize(); res < s && s <= blobSize { - res = s - } - } - return -} - -func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts openOpts) (*Reader, error) { - if tocOff < 0 { - // This means that TOC isn't contained in the blob. - // We pass nil reader to ParseTOC and expect that ParseTOC acquire TOC from - // the external location. - start := time.Now() - toc, tocDgst, err := d.ParseTOC(nil) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - if len(tocBytes) > 0 { - start := time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err == nil { - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - } - - start := time.Now() - tocBytes = make([]byte, tocSize) - if _, err := sr.ReadAt(tocBytes, tocOff); err != nil { - return nil, fmt.Errorf("error reading %d byte TOC targz: %v", len(tocBytes), err) - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - start = time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil -} - -func formatModtime(t time.Time) string { - if t.IsZero() || t.Unix() == 0 { - return "" - } - return t.UTC().Round(time.Second).Format(time.RFC3339) -} - -func cleanEntryName(name string) string { - // Use path.Clean to consistently deal with path separators across platforms. - return strings.TrimPrefix(path.Clean("/"+name), "/") -} - -// countWriter counts how many bytes have been written to its wrapped -// io.Writer. -type countWriter struct { - w io.Writer - n int64 -} - -func (cw *countWriter) Write(p []byte) (n int, err error) { - n, err = cw.w.Write(p) - cw.n += int64(n) - return -} - -type countWriteFlusher struct { - io.WriteCloser - n int64 -} - -func (wc *countWriteFlusher) register(w io.WriteCloser) io.WriteCloser { - wc.WriteCloser = w - return wc -} - -func (wc *countWriteFlusher) Write(p []byte) (n int, err error) { - n, err = wc.WriteCloser.Write(p) - wc.n += int64(n) - return -} - -func (wc *countWriteFlusher) Flush() error { - if f, ok := wc.WriteCloser.(interface { - Flush() error - }); ok { - return f.Flush() - } - return nil -} - -func (wc *countWriteFlusher) Close() error { - err := wc.WriteCloser.Close() - wc.WriteCloser = nil - return err -} - -// isGzip reports whether br is positioned right before an upcoming gzip stream. -// It does not consume any bytes from br. -func isGzip(br *bufio.Reader) bool { - const ( - gzipID1 = 0x1f - gzipID2 = 0x8b - gzipDeflate = 8 - ) - peek, _ := br.Peek(3) - return len(peek) >= 3 && peek[0] == gzipID1 && peek[1] == gzipID2 && peek[2] == gzipDeflate -} - -func positive(n int64) int64 { - if n < 0 { - return 0 - } - return n -} - -type countReader struct { - r io.Reader - n int64 -} - -func (cr *countReader) Read(p []byte) (n int, err error) { - n, err = cr.r.Read(p) - cr.n += int64(n) - return -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go deleted file mode 100644 index 88fa13b19..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ /dev/null @@ -1,237 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "encoding/binary" - "encoding/json" - "fmt" - "hash" - "io" - "strconv" - - digest "github.com/opencontainers/go-digest" -) - -type gzipCompression struct { - *GzipCompressor - *GzipDecompressor -} - -func newGzipCompressionWithLevel(level int) Compression { - return &gzipCompression{ - &GzipCompressor{level}, - &GzipDecompressor{}, - } -} - -func NewGzipCompressor() *GzipCompressor { - return &GzipCompressor{gzip.BestCompression} -} - -func NewGzipCompressorWithLevel(level int) *GzipCompressor { - return &GzipCompressor{level} -} - -type GzipCompressor struct { - compressionLevel int -} - -func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) { - return gzip.NewWriterLevel(w, gc.compressionLevel) -} - -func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) { - tocJSON, err := json.MarshalIndent(toc, "", "\t") - if err != nil { - return "", err - } - gz, _ := gzip.NewWriterLevel(w, gc.compressionLevel) - gw := io.Writer(gz) - if diffHash != nil { - gw = io.MultiWriter(gz, diffHash) - } - tw := tar.NewWriter(gw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - return "", err - } - if _, err := tw.Write(tocJSON); err != nil { - return "", err - } - - if err := tw.Close(); err != nil { - return "", err - } - if err := gz.Close(); err != nil { - return "", err - } - if _, err := w.Write(gzipFooterBytes(off)); err != nil { - return "", err - } - return digest.FromBytes(tocJSON), nil -} - -// gzipFooterBytes returns the 51 bytes footer. -func gzipFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - - // Extra header indicating the offset of TOCJSON - // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 - header := make([]byte, 4) - header[0], header[1] = 'S', 'G' - subfield := fmt.Sprintf("%016xSTARGZ", tocOff) - binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) - } - return buf.Bytes() -} - -type GzipDecompressor struct{} - -func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != FooterSize { - return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, err - } - defer zr.Close() - extra := zr.Extra - si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] - if si1 != 'S' || si2 != 'G' { - return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) - } - if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { - return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) - } - if string(subfield[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") - } - tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *GzipDecompressor) FooterSize() int64 { - return FooterSize -} - -func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -type LegacyGzipDecompressor struct{} - -func (gz *LegacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *LegacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != legacyFooterSize { - return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to get footer gzip reader: %w", err) - } - defer zr.Close() - extra := zr.Extra - if len(extra) != 16+len("STARGZ") { - return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") - } - if string(extra[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") - } - tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *LegacyGzipDecompressor) FooterSize() int64 { - return legacyFooterSize -} - -func (gz *LegacyGzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - tr, err := decompressTOCEStargz(r) - if err != nil { - return nil, "", err - } - dgstr := digest.Canonical.Digester() - toc = new(JTOC) - if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil { - return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err) - } - if err := tr.Close(); err != nil { - return nil, "", err - } - return toc, dgstr.Digest(), nil -} - -func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) { - zr, err := gzip.NewReader(r) - if err != nil { - return nil, fmt.Errorf("malformed TOC gzip header: %v", err) - } - zr.Multistream(false) - tr := tar.NewReader(zr) - h, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err) - } - if h.Name != TOCTarName { - return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName) - } - return readCloser{tr, zr.Close}, nil -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go deleted file mode 100644 index ff165e090..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ /dev/null @@ -1,2403 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto/rand" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "os" - "path/filepath" - "reflect" - "sort" - "strings" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" -) - -// TestingController is Compression with some helper methods necessary for testing. -type TestingController interface { - Compression - TestStreams(t TestingT, b []byte, streams []int64) - DiffIDOf(TestingT, []byte) string - String() string -} - -// TestingT is the minimal set of testing.T required to run the -// tests defined in CompressionTestSuite. This interface exists to prevent -// leaking the testing package from being exposed outside tests. -type TestingT interface { - Errorf(format string, args ...any) - FailNow() - Failed() bool - Fatal(args ...any) - Fatalf(format string, args ...any) - Logf(format string, args ...any) - Parallel() -} - -// Runner allows running subtests of TestingT. This exists instead of adding -// a Run method to TestingT interface because the Run implementation of -// testing.T would not satisfy the interface. -type Runner func(t TestingT, name string, fn func(t TestingT)) - -type TestRunner struct { - TestingT - Runner Runner -} - -func (r *TestRunner) Run(name string, run func(*TestRunner)) { - r.Runner(r.TestingT, name, func(t TestingT) { - run(&TestRunner{TestingT: t, Runner: r.Runner}) - }) -} - -// CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them. -func CompressionTestSuite(t *TestRunner, controllers ...TestingControllerFactory) { - t.Run("testBuild", func(t *TestRunner) { t.Parallel(); testBuild(t, controllers...) }) - t.Run("testDigestAndVerify", func(t *TestRunner) { - t.Parallel() - testDigestAndVerify(t, controllers...) - }) - t.Run("testWriteAndOpen", func(t *TestRunner) { t.Parallel(); testWriteAndOpen(t, controllers...) }) -} - -type TestingControllerFactory func() TestingController - -const ( - uncompressedType int = iota - gzipType - zstdType -) - -var srcCompressions = []int{ - uncompressedType, - gzipType, - zstdType, -} - -var allowedPrefix = [4]string{"", "./", "/", "../"} - -// testBuild tests the resulting stargz blob built by this pkg has the same -// contents as the normal stargz blob. -func testBuild(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - chunkSize int - minChunkSize []int - in []tarEntry - }{ - { - name: "regfiles and directories", - chunkSize: 4, - in: tarOf( - file("foo", "test1"), - dir("foo2/"), - file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})), - ), - }, - { - name: "empty files", - chunkSize: 4, - in: tarOf( - file("foo", "tttttt"), - file("foo_empty", ""), - file("foo2", "tttttt"), - file("foo_empty2", ""), - file("foo3", "tttttt"), - file("foo_empty3", ""), - file("foo4", "tttttt"), - file("foo_empty4", ""), - file("foo5", "tttttt"), - file("foo_empty5", ""), - file("foo6", "tttttt"), - ), - }, - { - name: "various files", - chunkSize: 4, - minChunkSize: []int{0, 64000}, - in: tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo1.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - { - name: "no contents", - chunkSize: 4, - in: tarOf( - file("baz.txt", ""), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - } - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, prefix := range allowedPrefix { - prefix := prefix - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) { - tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) - // Test divideEntries() - entries, err := sortEntries(tarBlob, nil, nil) // identical order - if err != nil { - t.Fatalf("failed to parse tar: %v", err) - } - var merged []*entry - for _, part := range divideEntries(entries, 4) { - merged = append(merged, part...) - } - if !reflect.DeepEqual(entries, merged) { - for _, e := range entries { - t.Logf("Original: %v", e.header) - } - for _, e := range merged { - t.Logf("Merged: %v", e.header) - } - t.Errorf("divided entries couldn't be merged") - return - } - - // Prepare sample data - cl1 := newCL() - wantBuf := new(bytes.Buffer) - sw := NewWriterWithCompressor(wantBuf, cl1) - sw.MinChunkSize = minChunkSize - sw.ChunkSize = tt.chunkSize - if err := sw.AppendTar(tarBlob); err != nil { - t.Fatalf("failed to append tar to want stargz: %v", err) - } - if _, err := sw.Close(); err != nil { - t.Fatalf("failed to prepare want stargz: %v", err) - } - wantData := wantBuf.Bytes() - want, err := Open(io.NewSectionReader( - bytes.NewReader(wantData), 0, int64(len(wantData))), - WithDecompressors(cl1), - ) - if err != nil { - t.Fatalf("failed to parse the want stargz: %v", err) - } - - // Prepare testing data - var opts []Option - if minChunkSize > 0 { - opts = append(opts, WithMinChunkSize(minChunkSize)) - } - cl2 := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - append(opts, WithChunkSize(tt.chunkSize), WithCompression(cl2))...) - if err != nil { - t.Fatalf("failed to build stargz: %v", err) - } - defer rc.Close() - gotBuf := new(bytes.Buffer) - if _, err := io.Copy(gotBuf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - gotData := gotBuf.Bytes() - got, err := Open(io.NewSectionReader( - bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), - WithDecompressors(cl2), - ) - if err != nil { - t.Fatalf("failed to parse the got stargz: %v", err) - } - - // Check DiffID is properly calculated - rc.Close() - diffID := rc.DiffID() - wantDiffID := cl2.DiffIDOf(t, gotData) - if diffID.String() != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - // Compare as stargz - if !isSameVersion(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz hasn't same json") - return - } - if !isSameEntries(t, want, got) { - t.Errorf("built stargz isn't same as the original") - return - } - - // Compare as tar.gz - if !isSameTarGz(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz isn't same tar.gz") - return - } - }) - } - } - } - } - } - } -} - -func isSameTarGz(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aGz, err := cla.Reader(bytes.NewReader(a)) - if err != nil { - t.Fatalf("failed to read A") - } - defer aGz.Close() - bGz, err := clb.Reader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("failed to read B") - } - defer bGz.Close() - - // Same as tar's Next() method but ignores landmarks and TOCJSON file - next := func(r *tar.Reader) (h *tar.Header, err error) { - for { - if h, err = r.Next(); err != nil { - return - } - if h.Name != PrefetchLandmark && - h.Name != NoPrefetchLandmark && - h.Name != TOCTarName { - return - } - } - } - - aTar := tar.NewReader(aGz) - bTar := tar.NewReader(bGz) - for { - // Fetch and parse next header. - aH, aErr := next(aTar) - bH, bErr := next(bTar) - if aErr != nil || bErr != nil { - if aErr == io.EOF && bErr == io.EOF { - break - } - t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr) - } - if !reflect.DeepEqual(aH, bH) { - t.Logf("different header (A = %v; B = %v)", aH, bH) - return false - - } - aFile, err := io.ReadAll(aTar) - if err != nil { - t.Fatal("failed to read tar payload of A") - } - bFile, err := io.ReadAll(bTar) - if err != nil { - t.Fatal("failed to read tar payload of B") - } - if !bytes.Equal(aFile, bFile) { - t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b)) - return false - } - } - - return true -} - -func isSameVersion(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), cla) - if err != nil { - t.Fatalf("failed to parse A: %v", err) - } - bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), clb) - if err != nil { - t.Fatalf("failed to parse B: %v", err) - } - t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, aJTOC)) - t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bJTOC)) - return aJTOC.Version == bJTOC.Version -} - -func isSameEntries(t TestingT, a, b *Reader) bool { - aroot, ok := a.Lookup("") - if !ok { - t.Fatalf("failed to get root of A") - } - broot, ok := b.Lookup("") - if !ok { - t.Fatalf("failed to get root of B") - } - aEntry := stargzEntry{aroot, a} - bEntry := stargzEntry{broot, b} - return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry) -} - -func compressBlob(t TestingT, src *io.SectionReader, srcCompression int) *io.SectionReader { - buf := new(bytes.Buffer) - var w io.WriteCloser - var err error - switch srcCompression { - case gzipType: - w = gzip.NewWriter(buf) - case zstdType: - w, err = zstd.NewWriter(buf) - if err != nil { - t.Fatalf("failed to init zstd writer: %v", err) - } - default: - return src - } - src.Seek(0, io.SeekStart) - if _, err := io.Copy(w, src); err != nil { - t.Fatalf("failed to compress source") - } - if err := w.Close(); err != nil { - t.Fatalf("failed to finalize compress source") - } - data := buf.Bytes() - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) - -} - -type stargzEntry struct { - e *TOCEntry - r *Reader -} - -// contains checks if all child entries in "b" are also contained in "a". -// This function also checks if the files/chunks contain the same contents among "a" and "b". -func contains(t TestingT, a, b stargzEntry) bool { - ae, ar := a.e, a.r - be, br := b.e, b.r - t.Logf("Comparing: %q vs %q", ae.Name, be.Name) - if !equalEntry(ae, be) { - t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be) - return false - } - if ae.Type == "dir" { - t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name, - allChildrenName(ae), allChildrenName(be)) - iscontain := true - ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool { - // Walk through all files on this stargz file. - - if aChild.Name == PrefetchLandmark || - aChild.Name == NoPrefetchLandmark { - return true // Ignore landmarks - } - - // Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory - // because this points to the root directory itself. - if aChild.Name == "" && ae.Name == "" { - return true - } - - bChild, ok := be.LookupChild(aBaseName) - if !ok { - t.Logf("%q (base: %q): not found in b: %v", - ae.Name, aBaseName, allChildrenName(be)) - iscontain = false - return false - } - - childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r}) - if !childcontain { - t.Logf("%q != %q: non-equal dir", ae.Name, be.Name) - iscontain = false - return false - } - return true - }) - return iscontain - } else if ae.Type == "reg" { - af, err := ar.OpenFile(ae.Name) - if err != nil { - t.Fatalf("failed to open file %q on A: %v", ae.Name, err) - } - bf, err := br.OpenFile(be.Name) - if err != nil { - t.Fatalf("failed to open file %q on B: %v", be.Name, err) - } - - var nr int64 - for nr < ae.Size { - abytes, anext, aok := readOffset(t, af, nr, a) - bbytes, bnext, bok := readOffset(t, bf, nr, b) - if !aok && !bok { - break - } else if !aok || !bok || anext != bnext { - t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v", - ae.Name, be.Name, nr, aok, bok, anext, bnext) - return false - } - nr = anext - if !bytes.Equal(abytes, bbytes) { - t.Logf("%q != %q: different contents %v vs %v", - ae.Name, be.Name, string(abytes), string(bbytes)) - return false - } - } - return true - } - - return true -} - -func allChildrenName(e *TOCEntry) (children []string) { - e.ForeachChild(func(baseName string, _ *TOCEntry) bool { - children = append(children, baseName) - return true - }) - return -} - -func equalEntry(a, b *TOCEntry) bool { - // Here, we selectively compare fileds that we are interested in. - return a.Name == b.Name && - a.Type == b.Type && - a.Size == b.Size && - a.ModTime3339 == b.ModTime3339 && - a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime time.Time - a.LinkName == b.LinkName && - a.Mode == b.Mode && - a.UID == b.UID && - a.GID == b.GID && - a.Uname == b.Uname && - a.Gname == b.Gname && - (a.Offset >= 0) == (b.Offset >= 0) && - (a.NextOffset() > 0) == (b.NextOffset() > 0) && - a.DevMajor == b.DevMajor && - a.DevMinor == b.DevMinor && - a.NumLink == b.NumLink && - reflect.DeepEqual(a.Xattrs, b.Xattrs) && - // chunk-related infomations aren't compared in this function. - // ChunkOffset int64 `json:"chunkOffset,omitempty"` - // ChunkSize int64 `json:"chunkSize,omitempty"` - // children map[string]*TOCEntry - a.Digest == b.Digest -} - -func readOffset(t TestingT, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) { - ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset) - if !ok { - return nil, 0, false - } - data := make([]byte, ce.ChunkSize) - t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset()) - n, err := r.ReadAt(data, ce.ChunkOffset) - if err != nil { - t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v", - e.e.Name, ce.ChunkOffset, ce.ChunkSize, err) - } - if int64(n) != ce.ChunkSize { - t.Fatalf("unexpected copied data size %d; want %d", - n, ce.ChunkSize) - } - return data[:n], offset + ce.ChunkSize, true -} - -func dumpTOCJSON(t TestingT, tocJSON *JTOC) string { - jtocData, err := json.Marshal(*tocJSON) - if err != nil { - t.Fatalf("failed to marshal TOC JSON: %v", err) - } - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil { - t.Fatalf("failed to read toc json blob: %v", err) - } - return buf.String() -} - -const chunkSize = 3 - -type check func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) - -// testDigestAndVerify runs specified checks against sample stargz blobs. -func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - tarInit func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) - checks []check - minChunkSize []int - }{ - { - name: "no-regfile", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - dir("test/"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - dir("test2/"), // modified - ), allowedPrefix[0])), - }, - }, - { - name: "small-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "bbb", dgstMap), - ) - }, - minChunkSize: []int{0, 64000}, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", ""), - file("foo.txt", "M"), // modified - dir("test/"), - file("test/bar.txt", "bbb"), - ), allowedPrefix[0])), - // checkVerifyInvalidTOCEntryFail("foo.txt"), // TODO - checkVerifyBrokenContentFail("foo.txt"), - }, - }, - { - name: "big-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified - file("foo.txt", "a"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - { - name: "with-non-regfiles", - minChunkSize: []int{0, 64000}, - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - regDigest(t, "bar/foo2.txt", "b", dgstMap), - regDigest(t, "foo3.txt", "c", dgstMap), - symlink("barlink", "test/bar.txt"), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - dir("test2/"), - link("test2/bazlink", "foo.txt"), // modified - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - } - - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) { - // Get original tar file and chunk digests - dgstMap := make(map[string]digest.Digest) - tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat) - - cl := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - tocDigest := rc.TOCDigest() - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - newStargz := buf.Bytes() - // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. - dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) - - for _, check := range tt.checks { - check(t, newStargz, tocDigest, dgstMap, cl, newCL) - } - }) - } - } - } - } - } - } -} - -// checkStargzTOC checks the TOC JSON of the passed stargz has the expected -// digest and contains valid chunks. It walks all entries in the stargz and -// checks all chunk digests stored to the TOC JSON match the actual contents. -func checkStargzTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - digestMapTOC, err := listDigests(io.NewSectionReader( - bytes.NewReader(sgzData), 0, int64(len(sgzData))), - controller, - ) - if err != nil { - t.Fatalf("failed to list digest: %v", err) - } - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - // Check the digest of TOC JSON based on the actual contents - // It's sure that TOC JSON exists in this archive because - // Open succeeded. - dgstr := digest.Canonical.Digester() - if _, err := io.Copy(dgstr.Hash(), tr); err != nil { - t.Fatalf("failed to calculate digest of TOC JSON: %v", - err) - } - if dgstr.Digest() != tocDigest { - t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest()) - } - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - // Get the original digest to make sure the file contents are kept unchanged - // from the original tar, during the whole conversion steps. - id := chunkID(h.Name, n, ce.ChunkSize) - want, ok := dgstMap[id] - if !ok { - t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v", - h.Name, n, ce.ChunkSize, dgstMap) - return - } - found[id] = true - - // Check the file contents - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if want != dgstr.Digest() { - t.Errorf("Invalid contents in converted stargz %q: %q; want %q", - h.Name, dgstr.Digest(), want) - return - } - - // Check the digest stored in TOC JSON - dgstTOC, ok := digestMapTOC[ce.Offset] - if !ok { - t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered", - h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset) - } - if want != dgstTOC { - t.Errorf("Invalid digest in TOCEntry %q: %q; want %q", - h.Name, dgstTOC, want) - return - } - - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyTOC checks the verification works for the TOC JSON of the passed -// stargz. It walks all entries in the stargz and checks the verifications for -// all chunks work. -func checkVerifyTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Errorf("failed to verify stargz: %v", err) - return - } - - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - v, err := ev.Verifier(ce) - if err != nil { - t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n) - } - - found[chunkID(h.Name, n, ce.ChunkSize)] = true - - // Check the file contents - if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if !v.Verified() { - t.Errorf("Invalid contents in converted stargz %q (should be succeeded)", - h.Name) - return - } - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be -// detected during the verification and the verification returns an error. -func checkVerifyInvalidTOCEntryFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - funcs := map[string]rewriteFunc{ - "lost digest in a entry": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var found bool - for _, e := range toc.Entries { - if cleanEntryName(e.Name) == filename { - if e.Type != "reg" && e.Type != "chunk" { - t.Fatalf("entry %q to break must be regfile or chunk", filename) - } - if e.ChunkDigest == "" { - t.Fatalf("entry %q is already invalid", filename) - } - e.ChunkDigest = "" - found = true - } - } - if !found { - t.Fatalf("rewrite target not found") - } - }, - "duplicated entry offset": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var ( - sampleEntry *TOCEntry - targetEntry *TOCEntry - ) - for _, e := range toc.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if cleanEntryName(e.Name) == filename { - targetEntry = e - } else { - sampleEntry = e - } - } - } - if sampleEntry == nil { - t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") - return - } - if targetEntry == nil { - t.Fatalf("rewrite target not found") - return - } - targetEntry.Offset = sampleEntry.Offset - }, - } - - for name, rFunc := range funcs { - t.Run(name, func(t *TestRunner) { - newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, controller) - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, newSgz); err != nil { - t.Fatalf("failed to get converted stargz") - } - isgz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(newTocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - }) - } - } -} - -// checkVerifyInvalidStargzFail checks if the verification detects that the -// given stargz file doesn't match to the expected digest and returns error. -func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - cl := newController() - rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - mStargz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))), - WithDecompressors(cl), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(tocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - } -} - -// checkVerifyBrokenContentFail checks if the verifier detects broken contents -// that doesn't match to the expected digest and returns error. -func checkVerifyBrokenContentFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - // Parse stargz file - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Fatalf("failed to verify stargz: %v", err) - return - } - - // Open the target file - sr, err := sgz.OpenFile(filename) - if err != nil { - t.Fatalf("failed to open file %q", filename) - } - ce, ok := sgz.ChunkEntryForOffset(filename, 0) - if !ok { - t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0) - return - } - if ce.ChunkSize == 0 { - t.Fatalf("file mustn't be empty") - return - } - data := make([]byte, ce.ChunkSize) - if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil { - t.Errorf("failed to get data of a chunk of %q(offset=%q)", - filename, ce.ChunkOffset) - } - - // Check the broken chunk (must fail) - v, err := ev.Verifier(ce) - if err != nil { - t.Fatalf("failed to get verifier for %q", filename) - } - broken := append([]byte{^data[0]}, data[1:]...) - if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - filename, ce.ChunkOffset, ce.ChunkSize) - } - if v.Verified() { - t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)", - filename, data, broken) - } - } -} - -func chunkID(name string, offset, size int64) string { - return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size) -} - -type rewriteFunc func(t TestingT, toc *JTOC, sgz *io.SectionReader) - -func rewriteTOCJSON(t TestingT, sgz *io.SectionReader, rewrite rewriteFunc, controller TestingController) (newSgz io.Reader, tocDigest digest.Digest) { - decodedJTOC, jtocOffset, err := parseStargz(sgz, controller) - if err != nil { - t.Fatalf("failed to extract TOC JSON: %v", err) - } - - rewrite(t, decodedJTOC, sgz) - - tocFooter, tocDigest, err := tocAndFooter(controller, decodedJTOC, jtocOffset) - if err != nil { - t.Fatalf("failed to create toc and footer: %v", err) - } - - // Reconstruct stargz file with the modified TOC JSON - if _, err := sgz.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to reset the seek position of stargz: %v", err) - } - return io.MultiReader( - io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON) - tocFooter, // Rewritten TOC and footer - ), tocDigest -} - -func listDigests(sgz *io.SectionReader, controller TestingController) (map[int64]digest.Digest, error) { - decodedJTOC, _, err := parseStargz(sgz, controller) - if err != nil { - return nil, err - } - digestMap := make(map[int64]digest.Digest) - for _, e := range decodedJTOC.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if e.Type == "reg" && e.Size == 0 { - continue // ignores empty file - } - if e.ChunkDigest == "" { - return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", - e.Name, e.Offset) - } - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, err - } - digestMap[e.Offset] = d - } - } - return digestMap, nil -} - -func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJTOC *JTOC, jtocOffset int64, err error) { - fSize := controller.FooterSize() - footer := make([]byte, fSize) - if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil { - return nil, 0, fmt.Errorf("error reading footer: %w", err) - } - _, tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse footer: %w", err) - } - - // Decode the TOC JSON - var tocReader io.Reader - if tocOffset >= 0 { - tocReader = io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize) - } - decodedJTOC, _, err = controller.ParseTOC(tocReader) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse TOC: %w", err) - } - return decodedJTOC, tocOffset, nil -} - -func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { - const content = "Some contents" - invalidUtf8 := "\xff\xfe\xfd" - - xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8} - sampleOwner := owner{uid: 50, gid: 100} - - data64KB := randomContents(64000) - - tests := []struct { - name string - chunkSize int - minChunkSize int - in []tarEntry - want []stargzCheck - wantNumGz int // expected number of streams - - wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz - wantFailOnLossLess bool - wantTOCVersion int // default = 1 - }{ - { - name: "empty", - in: tarOf(), - wantNumGz: 2, // (empty tar) + TOC + footer - want: checks( - numTOCEntries(0), - ), - }, - { - name: "1dir_1empty_file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", ""), - ), - wantNumGz: 3, // dir, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", 0), - entryHasChildren("foo", "bar.txt"), - hasFileDigest("foo/bar.txt", digestFor("")), - ), - }, - { - name: "1dir_1file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", content, xAttrFile), - ), - wantNumGz: 4, // var dir, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - hasFileDigest("foo/bar.txt", digestFor(content)), - hasFileContentsRange("foo/bar.txt", 0, content), - hasFileContentsRange("foo/bar.txt", 1, content[1:]), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar.txt"), - hasFileXattrs("foo/bar.txt", "foo", "bar"), - hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8), - ), - }, - { - name: "2meta_2file", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - }, - { - name: "3dir", - in: tarOf( - dir("bar/"), - dir("foo/"), - dir("foo/bar/"), - ), - wantNumGz: 3, // 3 dirs, TOC, footer - want: checks( - hasDirLinkCount("bar/", 2), - hasDirLinkCount("foo/", 3), - hasDirLinkCount("foo/bar/", 2), - ), - }, - { - name: "symlink", - in: tarOf( - dir("foo/"), - symlink("foo/bar", "../../x"), - ), - wantNumGz: 3, // metas + TOC + footer - want: checks( - numTOCEntries(2), - hasSymlink("foo/bar", "../../x"), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar"), - ), - }, - { - name: "chunked_file", - chunkSize: 4, - in: tarOf( - dir("foo/"), - file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"), - ), - wantNumGz: 9, // dir + big.txt(6 chunks) + TOC + footer - want: checks( - numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file - hasDir("foo/"), - hasFileLen("foo/big.txt", len("This is such a big file")), - hasFileDigest("foo/big.txt", digestFor("This is such a big file")), - hasFileContentsRange("foo/big.txt", 0, "This is such a big file"), - hasFileContentsRange("foo/big.txt", 1, "his is such a big file"), - hasFileContentsRange("foo/big.txt", 2, "is is such a big file"), - hasFileContentsRange("foo/big.txt", 3, "s is such a big file"), - hasFileContentsRange("foo/big.txt", 4, " is such a big file"), - hasFileContentsRange("foo/big.txt", 5, "is such a big file"), - hasFileContentsRange("foo/big.txt", 6, "s such a big file"), - hasFileContentsRange("foo/big.txt", 7, " such a big file"), - hasFileContentsRange("foo/big.txt", 8, "such a big file"), - hasFileContentsRange("foo/big.txt", 9, "uch a big file"), - hasFileContentsRange("foo/big.txt", 10, "ch a big file"), - hasFileContentsRange("foo/big.txt", 11, "h a big file"), - hasFileContentsRange("foo/big.txt", 12, " a big file"), - hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""), - hasChunkEntries("foo/big.txt", 6), - ), - }, - { - name: "recursive", - in: tarOf( - dir("/", sampleOwner), - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // dirs, bar.txt alone, TOC, footer - want: checks( - maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt" - ), - }, - { - name: "block_char_fifo", - in: tarOf( - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "b", - Typeflag: tar.TypeBlock, - Devmajor: 123, - Devminor: 456, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "c", - Typeflag: tar.TypeChar, - Devmajor: 111, - Devminor: 222, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "f", - Typeflag: tar.TypeFifo, - Format: format, - }) - }), - ), - wantNumGz: 3, - want: checks( - lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}), - lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}), - lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}), - ), - }, - { - name: "modes", - in: tarOf( - dir("foo1/", 0755|os.ModeDir|os.ModeSetgid), - file("foo1/bar1", content, 0700|os.ModeSetuid), - file("foo1/bar2", content, 0755|os.ModeSetgid), - dir("foo2/", 0755|os.ModeDir|os.ModeSticky), - file("foo2/bar3", content, 0755|os.ModeSticky), - dir("foo3/", 0755|os.ModeDir), - file("foo3/bar4", content, os.FileMode(0700)), - file("foo3/bar5", content, os.FileMode(0755)), - ), - wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer - want: checks( - hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid), - hasMode("foo1/bar1", 0700|os.ModeSetuid), - hasMode("foo1/bar2", 0755|os.ModeSetgid), - hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky), - hasMode("foo2/bar3", 0755|os.ModeSticky), - hasMode("foo3/", 0755|os.ModeDir), - hasMode("foo3/bar4", os.FileMode(0700)), - hasMode("foo3/bar5", os.FileMode(0755)), - ), - }, - { - name: "lossy", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - file(TOCTarName, "dummy"), // ignored by the writer. (lossless write returns error) - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - wantFailOnLossLess: true, - }, - { - name: "hardlink should be replaced to the destination entry", - in: tarOf( - dir("foo/"), - file("foo/foo1", "test"), - link("foolink", "foo/foo1"), - ), - wantNumGz: 4, // dir, foo1 + link, TOC, footer - want: checks( - mustSameEntry("foo/foo1", "foolink"), - ), - }, - { - name: "several_files_in_chunk", - minChunkSize: 8000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - file("foo22", "ccc"), - dir("bar/"), - file("bar/bar.txt", "aaa"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed "data64KB" is still larger than 8KB - wantNumGz: 4, // dir+foo1, foo2+foo22+dir+bar.txt+foo3, TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1, foo2, foo22, dir, bar.txt, foo3 - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo22", len("ccc")), - hasFileLen("bar/bar.txt", len("aaa")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo22", digestFor("ccc")), - hasFileDigest("bar/bar.txt", digestFor("aaa")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo22", 0, "ccc", chunkInfo{"foo2", "bb"}, chunkInfo{"bar/bar.txt", "aaa"}, chunkInfo{"foo3", data64KB}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo22", 0, "ccc"), - hasFileContentsRange("foo22", 1, "cc"), - hasFileContentsRange("foo22", 2, "c"), - hasFileContentsRange("bar/bar.txt", 0, "aaa"), - hasFileContentsRange("bar/bar.txt", 1, "aa"), - hasFileContentsRange("bar/bar.txt", 2, "a"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - { - name: "several_files_in_chunk_chunked", - minChunkSize: 8000, - chunkSize: 32000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - dir("bar/"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed chunk of "data64KB" is still larger than 8KB - wantNumGz: 6, // dir+foo1(1), foo1(2), foo2+dir+foo3(1), foo3(2), TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1(2 chunks), foo2, dir, foo3(2 chunks) - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo2", 0, "bb", chunkInfo{"foo3", data64KB[:32000]}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo/foo1", 1, data64KB[1:]), - hasFileContentsRange("foo/foo1", 2, data64KB[2:]), - hasFileContentsRange("foo/foo1", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo/foo1", len(data64KB)-1, data64KB[len(data64KB)-1:]), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - } - - for _, tt := range tests { - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, lossless := range []bool{true, false} { - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) { - var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) - origTarDgstr := digest.Canonical.Digester() - tr = io.TeeReader(tr, origTarDgstr.Hash()) - var stargzBuf bytes.Buffer - cl1 := newCL() - w := NewWriterWithCompressor(&stargzBuf, cl1) - w.ChunkSize = tt.chunkSize - w.MinChunkSize = tt.minChunkSize - if lossless { - err := w.AppendTarLossLess(tr) - if tt.wantFailOnLossLess { - if err != nil { - return // expected to fail - } - t.Fatalf("Append wanted to fail on lossless") - } - if err != nil { - t.Fatalf("Append(lossless): %v", err) - } - } else { - if err := w.AppendTar(tr); err != nil { - t.Fatalf("Append: %v", err) - } - } - if _, err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - b := stargzBuf.Bytes() - - if lossless { - // Check if the result blob reserves original tar metadata - rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl1) - if err != nil { - t.Errorf("failed to decompress blob: %v", err) - return - } - defer rc.Close() - resultDgstr := digest.Canonical.Digester() - if _, err := io.Copy(resultDgstr.Hash(), rc); err != nil { - t.Errorf("failed to read result decompressed blob: %v", err) - return - } - if resultDgstr.Digest() != origTarDgstr.Digest() { - t.Errorf("lossy compression occurred: digest=%v; want %v", - resultDgstr.Digest(), origTarDgstr.Digest()) - return - } - } - - diffID := w.DiffID() - wantDiffID := cl1.DiffIDOf(t, b) - if diffID != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - telemetry, checkCalled := newCalledTelemetry() - sr := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))) - r, err := Open( - sr, - WithDecompressors(cl1), - WithTelemetry(telemetry), - ) - if err != nil { - t.Fatalf("stargz.Open: %v", err) - } - if _, ok := r.Lookup(""); !ok { - t.Fatalf("failed to lookup rootdir: %v", err) - } - wantTOCVersion := 1 - if tt.wantTOCVersion > 0 { - wantTOCVersion = tt.wantTOCVersion - } - if r.toc.Version != wantTOCVersion { - t.Fatalf("invalid TOC Version %d; wanted %d", r.toc.Version, wantTOCVersion) - } - - footerSize := cl1.FooterSize() - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - t.Errorf("failed to read footer: %v", err) - } - _, tocOffset, _, err := cl1.ParseFooter(footer) - if err != nil { - t.Errorf("failed to parse footer: %v", err) - } - if err := checkCalled(tocOffset >= 0); err != nil { - t.Errorf("telemetry failure: %v", err) - } - - wantNumGz := tt.wantNumGz - if lossless && tt.wantNumGzLossLess > 0 { - wantNumGz = tt.wantNumGzLossLess - } - streamOffsets := []int64{0} - prevOffset := int64(-1) - streams := 0 - for _, e := range r.toc.Entries { - if e.Offset > prevOffset { - streamOffsets = append(streamOffsets, e.Offset) - prevOffset = e.Offset - streams++ - } - } - streams++ // TOC - if tocOffset >= 0 { - // toc is in the blob - streamOffsets = append(streamOffsets, tocOffset) - } - streams++ // footer - streamOffsets = append(streamOffsets, footerOffset) - if streams != wantNumGz { - t.Errorf("number of streams in TOC = %d; want %d", streams, wantNumGz) - } - - t.Logf("testing streams: %+v", streamOffsets) - cl1.TestStreams(t, b, streamOffsets) - - for _, want := range tt.want { - want.check(t, r) - } - }) - } - } - } - } - } -} - -type chunkInfo struct { - name string - data string -} - -func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) error) { - var getFooterLatencyCalled bool - var getTocLatencyCalled bool - var deserializeTocLatencyCalled bool - return &Telemetry{ - func(time.Time) { getFooterLatencyCalled = true }, - func(time.Time) { getTocLatencyCalled = true }, - func(time.Time) { deserializeTocLatencyCalled = true }, - }, func(needsGetTOC bool) error { - var allErr []error - if !getFooterLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called")) - } - if needsGetTOC { - if !getTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called")) - } - } - if !deserializeTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) - } - return errorutil.Aggregate(allErr) - } -} - -func digestFor(content string) string { - sum := sha256.Sum256([]byte(content)) - return fmt.Sprintf("sha256:%x", sum) -} - -type numTOCEntries int - -func (n numTOCEntries) check(t TestingT, r *Reader) { - if r.toc == nil { - t.Fatal("nil TOC") - } - if got, want := len(r.toc.Entries), int(n); got != want { - t.Errorf("got %d TOC entries; want %d", got, want) - } - t.Logf("got TOC entries:") - for i, ent := range r.toc.Entries { - entj, _ := json.Marshal(ent) - t.Logf(" [%d]: %s\n", i, entj) - } - if t.Failed() { - t.FailNow() - } -} - -func checks(s ...stargzCheck) []stargzCheck { return s } - -type stargzCheck interface { - check(t TestingT, r *Reader) -} - -type stargzCheckFn func(TestingT, *Reader) - -func (f stargzCheckFn) check(t TestingT, r *Reader) { f(t, r) } - -func maxDepth(max int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup("") - if !ok { - t.Fatal("root directory not found") - } - d, err := getMaxDepth(t, e, 0, 10*max) - if err != nil { - t.Errorf("failed to get max depth (wanted %d): %v", max, err) - return - } - if d != max { - t.Errorf("invalid depth %d; want %d", d, max) - return - } - }) -} - -func getMaxDepth(t TestingT, e *TOCEntry, current, limit int) (max int, rErr error) { - if current > limit { - return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d", - current, limit) - } - max = current - e.ForeachChild(func(baseName string, ent *TOCEntry) bool { - t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name) - d, err := getMaxDepth(t, ent, current+1, limit) - if err != nil { - rErr = err - return false - } - if d > max { - max = d - } - return true - }) - return -} - -func hasFileLen(file string, wantLen int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } else if ent.Size != int64(wantLen) { - t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen) - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileXattrs(file, name, value string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } - if ent.Xattrs == nil { - t.Errorf("file %q has no xattrs", file) - return - } - valueFound, found := ent.Xattrs[name] - if !found { - t.Errorf("file %q has no xattr %q", file, name) - return - } - if string(valueFound) != value { - t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value) - } - - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileDigest(file string, digest string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("didn't find TOCEntry for file %q", file) - } - if ent.Digest != digest { - t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest) - } - }) -} - -func hasFileContentsWithPreRead(file string, offset int, want string, extra ...chunkInfo) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - extraMap := make(map[string]chunkInfo) - for _, e := range extra { - extraMap[e.name] = e - } - var extraNames []string - for n := range extraMap { - extraNames = append(extraNames, n) - } - f, err := r.OpenFileWithPreReader(file, func(e *TOCEntry, cr io.Reader) error { - t.Logf("On %q: got preread of %q", file, e.Name) - ex, ok := extraMap[e.Name] - if !ok { - t.Fatalf("fail on %q: unexpected entry %q: %+v, %+v", file, e.Name, e, extraNames) - } - got, err := io.ReadAll(cr) - if err != nil { - t.Fatalf("fail on %q: failed to read %q: %v", file, e.Name, err) - } - if ex.data != string(got) { - t.Fatalf("fail on %q: unexpected contents of %q: len=%d; want=%d", file, e.Name, len(got), len(ex.data)) - } - delete(extraMap, e.Name) - return nil - }) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d, size %d) = %v, %v", len(got), offset, f.Size(), n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - if len(extraMap) != 0 { - var exNames []string - for _, ex := range extraMap { - exNames = append(exNames, ex.name) - } - t.Fatalf("fail on %q: some entries aren't read: %+v", file, exNames) - } - }) -} - -func hasFileContentsRange(file string, offset int, want string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - f, err := r.OpenFile(file) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - }) -} - -func hasChunkEntries(file string, wantChunks int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("no file for %q", file) - } - if ent.Type != "reg" { - t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type) - } - chunks := r.getChunks(ent) - if len(chunks) != wantChunks { - t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks) - return - } - f := chunks[0] - - var gotChunks []*TOCEntry - var last *TOCEntry - for off := int64(0); off < f.Size; off++ { - e, ok := r.ChunkEntryForOffset(file, off) - if !ok { - t.Errorf("no ChunkEntryForOffset at %d", off) - return - } - if last != e { - gotChunks = append(gotChunks, e) - last = e - } - } - if !reflect.DeepEqual(chunks, gotChunks) { - t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks) - } - - // And verify the NextOffset - for i := 0; i < len(gotChunks)-1; i++ { - ci := gotChunks[i] - cnext := gotChunks[i+1] - if ci.NextOffset() != cnext.Offset { - t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset) - } - } - }) -} - -func entryHasChildren(dir string, want ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - want := append([]string(nil), want...) - var got []string - ent, ok := r.Lookup(dir) - if !ok { - t.Fatalf("didn't find TOCEntry for dir node %q", dir) - } - for baseName := range ent.children { - got = append(got, baseName) - } - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { - t.Errorf("children of %q = %q; want %q", dir, got, want) - } - }) -} - -func hasDir(file string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasDirLinkCount(file string, count int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - return - } - if ent.NumLink != count { - t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasMode(file string, mode os.FileMode) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Stat().Mode() != mode { - t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode) - return - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasSymlink(file, target string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "symlink" { - t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type) - } else if ent.LinkName != target { - t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target) - } - return - } - } - t.Errorf("symlink %q not found", file) - }) -} - -func lookupMatch(name string, want *TOCEntry) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup(name) - if !ok { - t.Fatalf("failed to Lookup entry %q", name) - } - if !reflect.DeepEqual(e, want) { - t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want) - } - - }) -} - -func hasEntryOwner(entry string, owner owner) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(strings.TrimSuffix(entry, "/")) - if !ok { - t.Errorf("entry %q not found", entry) - return - } - if ent.UID != owner.uid || ent.GID != owner.gid { - t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid) - return - } - }) -} - -func mustSameEntry(files ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - var first *TOCEntry - for _, f := range files { - if first == nil { - var ok bool - first, ok = r.Lookup(f) - if !ok { - t.Errorf("unknown first file on Lookup: %q", f) - return - } - } - - // Test Lookup - e, ok := r.Lookup(f) - if !ok { - t.Errorf("unknown file on Lookup: %q", f) - return - } - if e != first { - t.Errorf("Lookup: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test LookupChild - pe, ok := r.Lookup(filepath.Dir(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get parent of %q", f) - return - } - e, ok = pe.LookupChild(filepath.Base(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get %q as the child of %+v", f, pe) - return - } - if e != first { - t.Errorf("LookupChild: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test ForeachChild - pe.ForeachChild(func(baseName string, e *TOCEntry) bool { - if baseName == filepath.Base(filepath.Clean(f)) { - if e != first { - t.Errorf("ForeachChild: %+v(%p) != %+v(%p)", e, e, first, first) - return false - } - } - return true - }) - } - }) -} - -func viewContent(c []byte) string { - if len(c) < 100 { - return string(c) - } - return string(c[:50]) + "...(omit)..." + string(c[50:100]) -} - -func tarOf(s ...tarEntry) []tarEntry { return s } - -type tarEntry interface { - appendTar(tw *tar.Writer, prefix string, format tar.Format) error -} - -type tarEntryFunc func(*tar.Writer, string, tar.Format) error - -func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format) error { - return f(tw, prefix, format) -} - -func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { - format := tar.FormatUnknown - for _, opt := range opts { - switch v := opt.(type) { - case tar.Format: - format = v - default: - panic(fmt.Errorf("unsupported opt for buildTar: %v", opt)) - } - } - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix, format); err != nil { - t.Fatalf("building input tar: %v", err) - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - data := append(buf.Bytes(), make([]byte, 100)...) // append empty bytes at the tail to see lossless works - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) -} - -func dir(name string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var o owner - mode := os.FileMode(0755) - for _, opt := range opts { - switch v := opt.(type) { - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if !strings.HasSuffix(name, "/") { - panic(fmt.Sprintf("missing trailing slash in dir %q ", name)) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeDir, - Name: prefix + name, - Mode: tm, - Uid: o.uid, - Gid: o.gid, - Format: format, - }) - }) -} - -// xAttr are extended attributes to set on test files created with the file func. -type xAttr map[string]string - -// owner is owner ot set on test files and directories with the file and dir functions. -type owner struct { - uid int - gid int -} - -func file(name, contents string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var xattrs xAttr - var o owner - mode := os.FileMode(0644) - for _, opt := range opts { - switch v := opt.(type) { - case xAttr: - xattrs = v - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if strings.HasSuffix(name, "/") { - return fmt.Errorf("bogus trailing slash in file %q", name) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - if len(xattrs) > 0 { - format = tar.FormatPAX // only PAX supports xattrs - } - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Mode: tm, - Xattrs: xattrs, - Size: int64(len(contents)), - Uid: o.uid, - Gid: o.gid, - Format: format, - }); err != nil { - return err - } - _, err = io.WriteString(tw, contents) - return err - }) -} - -func symlink(name, target string) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeSymlink, - Name: prefix + name, - Linkname: target, - Mode: 0644, - Format: format, - }) - }) -} - -func link(name string, linkname string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeLink, - Name: prefix + name, - Linkname: linkname, - ModTime: now, - Format: format, - }) - }) -} - -func chardev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeChar, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} - -func blockdev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeBlock, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} -func fifo(name string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeFifo, - Name: prefix + name, - ModTime: now, - Format: format, - }) - }) -} - -func prefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func noPrefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func regDigest(t TestingT, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry { - if digestMap == nil { - t.Fatalf("digest map mustn't be nil") - } - content := []byte(contentStr) - - var n int64 - for n < int64(len(content)) { - size := int64(chunkSize) - remain := int64(len(content)) - n - if remain < size { - size = remain - } - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil { - t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)", - string(content[n:n+size]), name, n, size) - } - digestMap[chunkID(name, n, size)] = dgstr.Digest() - n += size - } - - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Size: int64(len(content)), - Format: format, - }); err != nil { - return err - } - if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil { - return err - } - return nil - }) -} - -var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randomContents(n int) string { - b := make([]rune, n) - for i := range b { - bi, err := rand.Int(rand.Reader, big.NewInt(int64(len(runes)))) - if err != nil { - panic(err) - } - b[i] = runes[int(bi.Int64())] - } - return string(b) -} - -func fileModeToTarMode(mode os.FileMode) (int64, error) { - h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "") - if err != nil { - return 0, err - } - return h.Mode, nil -} - -// fileInfoOnlyMode is os.FileMode that populates only file mode. -type fileInfoOnlyMode os.FileMode - -func (f fileInfoOnlyMode) Name() string { return "" } -func (f fileInfoOnlyMode) Size() int64 { return 0 } -func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } -func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } -func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } - -func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) { - if len(streams) == 0 { - return // nop - } - - wants := map[int64]struct{}{} - for _, s := range streams { - wants[s] = struct{}{} - } - - len0 := len(b) - br := bytes.NewReader(b) - zr := new(gzip.Reader) - t.Logf("got gzip streams:") - numStreams := 0 - for { - zoff := len0 - br.Len() - if err := zr.Reset(br); err != nil { - if err == io.EOF { - return - } - t.Fatalf("countStreams(gzip), Reset: %v", err) - } - zr.Multistream(false) - n, err := io.Copy(io.Discard, zr) - if err != nil { - t.Fatalf("countStreams(gzip), Copy: %v", err) - } - var extra string - if len(zr.Extra) > 0 { - extra = fmt.Sprintf("; extra=%q", zr.Extra) - } - t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra) - delete(wants, int64(zoff)) - numStreams++ - } -} - -func GzipDiffIDOf(t TestingT, b []byte) string { - h := sha256.New() - zr, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("diffIDOf(gzip): %v", err) - } - defer zr.Close() - if _, err := io.Copy(h, zr); err != nil { - t.Fatalf("diffIDOf(gzip).Copy: %v", err) - } - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go deleted file mode 100644 index 57e0aa614..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ /dev/null @@ -1,342 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "hash" - "io" - "os" - "path" - "time" - - digest "github.com/opencontainers/go-digest" -) - -const ( - // TOCTarName is the name of the JSON file in the tar archive in the - // table of contents gzip stream. - TOCTarName = "stargz.index.json" - - // FooterSize is the number of bytes in the footer - // - // The footer is an empty gzip stream with no compression and an Extra - // header of the form "%016xSTARGZ", where the 64 bit hex-encoded - // number is the offset to the gzip stream of JSON TOC. - // - // 51 comes from: - // - // 10 bytes gzip header - // 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ")) - // 2 bytes Extra: SI1 = 'S', SI2 = 'G' - // 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ")) - // 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC) - // 5 bytes flate header - // 8 bytes gzip footer - // (End of the eStargz blob) - // - // NOTE: For Extra fields, subfield IDs SI1='S' SI2='G' is used for eStargz. - FooterSize = 51 - - // legacyFooterSize is the number of bytes in the legacy stargz footer. - // - // 47 comes from: - // - // 10 byte gzip header + - // 2 byte (LE16) length of extra, encoding 22 (16 hex digits + len("STARGZ")) == "\x16\x00" + - // 22 bytes of extra (fmt.Sprintf("%016xSTARGZ", tocGzipOffset)) - // 5 byte flate header - // 8 byte gzip footer (two little endian uint32s: digest, size) - legacyFooterSize = 47 - - // TOCJSONDigestAnnotation is an annotation for an image layer. This stores the - // digest of the TOC JSON. - // This annotation is valid only when it is specified in `.[]layers.annotations` - // of an image manifest. - TOCJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest" - - // StoreUncompressedSizeAnnotation is an additional annotation key for eStargz to enable lazy - // pulling on containers/storage. Stargz Store is required to expose the layer's uncompressed size - // to the runtime but current OCI image doesn't ship this information by default. So we store this - // to the special annotation. - StoreUncompressedSizeAnnotation = "io.containers.estargz.uncompressed-size" - - // PrefetchLandmark is a file entry which indicates the end position of - // prefetch in the stargz file. - PrefetchLandmark = ".prefetch.landmark" - - // NoPrefetchLandmark is a file entry which indicates that no prefetch should - // occur in the stargz file. - NoPrefetchLandmark = ".no.prefetch.landmark" - - landmarkContents = 0xf -) - -// JTOC is the JSON-serialized table of contents index of the files in the stargz file. -type JTOC struct { - Version int `json:"version"` - Entries []*TOCEntry `json:"entries"` -} - -// TOCEntry is an entry in the stargz file's TOC (Table of Contents). -type TOCEntry struct { - // Name is the tar entry's name. It is the complete path - // stored in the tar file, not just the base name. - Name string `json:"name"` - - // Type is one of "dir", "reg", "symlink", "hardlink", "char", - // "block", "fifo", or "chunk". - // The "chunk" type is used for regular file data chunks past the first - // TOCEntry; the 2nd chunk and on have only Type ("chunk"), Offset, - // ChunkOffset, and ChunkSize populated. - Type string `json:"type"` - - // Size, for regular files, is the logical size of the file. - Size int64 `json:"size,omitempty"` - - // ModTime3339 is the modification time of the tar entry. Empty - // means zero or unknown. Otherwise it's in UTC RFC3339 - // format. Use the ModTime method to access the time.Time value. - ModTime3339 string `json:"modtime,omitempty"` - modTime time.Time - - // LinkName, for symlinks and hardlinks, is the link target. - LinkName string `json:"linkName,omitempty"` - - // Mode is the permission and mode bits. - Mode int64 `json:"mode,omitempty"` - - // UID is the user ID of the owner. - UID int `json:"uid,omitempty"` - - // GID is the group ID of the owner. - GID int `json:"gid,omitempty"` - - // Uname is the username of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same UID. - Uname string `json:"userName,omitempty"` - - // Gname is the group name of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same GID. - Gname string `json:"groupName,omitempty"` - - // Offset, for regular files, provides the offset in the - // stargz file to the file's data bytes. See ChunkOffset and - // ChunkSize. - Offset int64 `json:"offset,omitempty"` - - // InnerOffset is an optional field indicates uncompressed offset - // of this "reg" or "chunk" payload in a stream starts from Offset. - // This field enables to put multiple "reg" or "chunk" payloads - // in one chunk with having the same Offset but different InnerOffset. - InnerOffset int64 `json:"innerOffset,omitempty"` - - nextOffset int64 // the Offset of the next entry with a non-zero Offset - - // DevMajor is the major device number for "char" and "block" types. - DevMajor int `json:"devMajor,omitempty"` - - // DevMinor is the major device number for "char" and "block" types. - DevMinor int `json:"devMinor,omitempty"` - - // NumLink is the number of entry names pointing to this entry. - // Zero means one name references this entry. - // This field is calculated during runtime and not recorded in TOC JSON. - NumLink int `json:"-"` - - // Xattrs are the extended attribute for the entry. - Xattrs map[string][]byte `json:"xattrs,omitempty"` - - // Digest stores the OCI checksum for regular files payload. - // It has the form "sha256:abcdef01234....". - Digest string `json:"digest,omitempty"` - - // ChunkOffset is non-zero if this is a chunk of a large, - // regular file. If so, the Offset is where the gzip header of - // ChunkSize bytes at ChunkOffset in Name begin. - // - // In serialized form, a "chunkSize" JSON field of zero means - // that the chunk goes to the end of the file. After reading - // from the stargz TOC, though, the ChunkSize is initialized - // to a non-zero file for when Type is either "reg" or - // "chunk". - ChunkOffset int64 `json:"chunkOffset,omitempty"` - ChunkSize int64 `json:"chunkSize,omitempty"` - - // ChunkDigest stores an OCI digest of the chunk. This must be formed - // as "sha256:0123abcd...". - ChunkDigest string `json:"chunkDigest,omitempty"` - - children map[string]*TOCEntry - - // chunkTopIndex is index of the entry where Offset starts in the blob. - chunkTopIndex int -} - -// ModTime returns the entry's modification time. -func (e *TOCEntry) ModTime() time.Time { return e.modTime } - -// NextOffset returns the position (relative to the start of the -// stargz file) of the next gzip boundary after e.Offset. -func (e *TOCEntry) NextOffset() int64 { return e.nextOffset } - -func (e *TOCEntry) addChild(baseName string, child *TOCEntry) { - if e.children == nil { - e.children = make(map[string]*TOCEntry) - } - if child.Type == "dir" { - e.NumLink++ // Entry ".." in the subdirectory links to this directory - } - e.children[baseName] = child -} - -// isDataType reports whether TOCEntry is a regular file or chunk (something that -// contains regular file data). -func (e *TOCEntry) isDataType() bool { return e.Type == "reg" || e.Type == "chunk" } - -// Stat returns a FileInfo value representing e. -func (e *TOCEntry) Stat() os.FileInfo { return fileInfo{e} } - -// ForeachChild calls f for each child item. If f returns false, iteration ends. -// If e is not a directory, f is not called. -func (e *TOCEntry) ForeachChild(f func(baseName string, ent *TOCEntry) bool) { - for name, ent := range e.children { - if !f(name, ent) { - return - } - } -} - -// LookupChild returns the directory e's child by its base name. -func (e *TOCEntry) LookupChild(baseName string) (child *TOCEntry, ok bool) { - child, ok = e.children[baseName] - return -} - -// fileInfo implements os.FileInfo using the wrapped *TOCEntry. -type fileInfo struct{ e *TOCEntry } - -var _ os.FileInfo = fileInfo{} - -func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } -func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } -func (fi fileInfo) Size() int64 { return fi.e.Size } -func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() } -func (fi fileInfo) Sys() interface{} { return fi.e } -func (fi fileInfo) Mode() (m os.FileMode) { - // TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg. - m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() & - (os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky) - switch fi.e.Type { - case "dir": - m |= os.ModeDir - case "symlink": - m |= os.ModeSymlink - case "char": - m |= os.ModeDevice | os.ModeCharDevice - case "block": - m |= os.ModeDevice - case "fifo": - m |= os.ModeNamedPipe - } - return m -} - -// TOCEntryVerifier holds verifiers that are usable for verifying chunks contained -// in a eStargz blob. -type TOCEntryVerifier interface { - - // Verifier provides a content verifier that can be used for verifying the - // contents of the specified TOCEntry. - Verifier(ce *TOCEntry) (digest.Verifier, error) -} - -// Compression provides the compression helper to be used creating and parsing eStargz. -// This package provides gzip-based Compression by default, but any compression -// algorithm (e.g. zstd) can be used as long as it implements Compression. -type Compression interface { - Compressor - Decompressor -} - -// Compressor represents the helper mothods to be used for creating eStargz. -type Compressor interface { - // Writer returns WriteCloser to be used for writing a chunk to eStargz. - // Everytime a chunk is written, the WriteCloser is closed and Writer is - // called again for writing the next chunk. - // - // The returned writer should implement "Flush() error" function that flushes - // any pending compressed data to the underlying writer. - Writer(w io.Writer) (WriteFlushCloser, error) - - // WriteTOCAndFooter is called to write JTOC to the passed Writer. - // diffHash calculates the DiffID (uncompressed sha256 hash) of the blob - // WriteTOCAndFooter can optionally write anything that affects DiffID calculation - // (e.g. uncompressed TOC JSON). - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob when it's parsed. - WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error) -} - -// Decompressor represents the helper mothods to be used for parsing eStargz. -type Decompressor interface { - // Reader returns ReadCloser to be used for decompressing file payload. - Reader(r io.Reader) (io.ReadCloser, error) - - // FooterSize returns the size of the footer of this blob. - FooterSize() int64 - - // ParseFooter parses the footer and returns the offset and (compressed) size of TOC. - // payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between - // the top until the TOC JSON). - // - // If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader - // to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it. - // - // tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the - // footer (blob size - tocOff - FooterSize). - // If blobPayloadSize < 0, blobPayloadSize become the blob size. - ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) - - // ParseTOC parses TOC from the passed reader. The reader provides the partial contents - // of the underlying blob that has the range specified by ParseFooter method. - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob. This must match to the value returned from - // Compressor.WriteTOCAndFooter that is used when creating this blob. - // - // If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob. - // Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location - // and return it. - ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) -} - -type WriteFlushCloser interface { - io.WriteCloser - Flush() error -} diff --git a/vendor/github.com/docker/cli/AUTHORS b/vendor/github.com/docker/cli/AUTHORS index 57af08b20..accbf6c52 100644 --- a/vendor/github.com/docker/cli/AUTHORS +++ b/vendor/github.com/docker/cli/AUTHORS @@ -2,6 +2,7 @@ # This file lists all contributors to the repository. # See scripts/docs/generate-authors.sh to make modifications. +4RH1T3CT0R7 A. Lester Buck III Aanand Prasad Aaron L. Xu @@ -42,6 +43,7 @@ Alexander Larsson Alexander Morozov Alexander Ryabov Alexandre González +Alexandre Vallières-Lagacé Alexey Igrychev Alexis Couvreur Alfred Landrum @@ -64,6 +66,7 @@ Andres G. Aragoneses Andres Leon Rangel Andrew France Andrew He +Andrew Hopp Andrew Hsu Andrew Macpherson Andrew McDonnell @@ -127,6 +130,7 @@ Brian Goff Brian Tracy Brian Wieder Bruno Sousa +Bruno Verachten Bryan Bess Bryan Boreham Bryan Murphy @@ -178,6 +182,7 @@ Christopher Svensson Christy Norman Chun Chen Clinton Kitson +Codex Coenraad Loubser Colin Hebert Collin Guarino @@ -234,6 +239,7 @@ David Sheets David Williamson David Xia David Young +Davlat Davydov Deng Guangxing Denis Defreyne Denis Gladkikh @@ -241,6 +247,7 @@ Denis Ollier Dennis Docter dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Derek McGowan +Derek Misler Des Preston Deshi Xiao Dharmit Shah @@ -260,6 +267,7 @@ Dominik Braun Don Kjer Dong Chen DongGeon Lee +Dorin Geman Doug Davis Drew Erny Ed Costello @@ -358,7 +366,7 @@ Hugo Gabriel Eyherabide huqun Huu Nguyen Hyzhou Zhy -Iain MacDonald +Iain MacDonald Iain Samuel McLean Elder Ian Campbell Ian Philpot @@ -471,6 +479,7 @@ Justyn Temme Jyrki Puttonen Jérémie Drouet Jérôme Petazzoni +Jörg Sommer Jörg Thalheim Kai Blin Kai Qiang Wu (Kennan) @@ -539,10 +548,12 @@ Lovekesh Kumar Luca Favatella Luca Marturana Lucas Chan +Ludovic Temgoua Abanda Luis Henrique Mulinari Luka Hartwig Lukas Heeren Lukasz Zajaczkowski +Luo Jiyin Lydell Manganti Lénaïc Huard Ma Shimiao @@ -603,6 +614,7 @@ Michael Spetsiotis Michael Steinert Michael Tews Michael West +Michael Zampani Michal Minář Michał Czeraszkiewicz Miguel Angel Alvarez Cabrerizo @@ -617,6 +629,7 @@ Mike Goelzer Mike MacCana mikelinjie <294893458@qq.com> Mikhail Vasin +Milas Bowman Milind Chawre Mindaugas Rukas Miroslav Gula @@ -887,6 +900,7 @@ Vincent Batts Vincent Bernat Vincent Demeester Vincent Woo +Vineet Kumar Vishnu Kannan Vivek Goyal Wang Jie @@ -916,6 +930,7 @@ Yanqiang Miao Yassine Tijani Yi EungJun Ying Li +Yoan Wainmann Yong Tang Yosef Fertel Yu Peng diff --git a/vendor/github.com/docker/cli/cli/config/configfile/file.go b/vendor/github.com/docker/cli/cli/config/configfile/file.go index fab3ed4cb..1a0e5b46c 100644 --- a/vendor/github.com/docker/cli/cli/config/configfile/file.go +++ b/vendor/github.com/docker/cli/cli/config/configfile/file.go @@ -1,3 +1,6 @@ +// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: +//go:build go1.25 + package configfile import ( @@ -6,6 +9,7 @@ import ( "errors" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -374,9 +378,7 @@ func getConfiguredCredentialStore(c *ConfigFile, registryHostname string) string func (configFile *ConfigFile) GetAllCredentials() (map[string]types.AuthConfig, error) { auths := make(map[string]types.AuthConfig) addAll := func(from map[string]types.AuthConfig) { - for reg, ac := range from { - auths[reg] = ac - } + maps.Copy(auths, from) } defaultStore := configFile.GetCredentialsStore("") diff --git a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go index c69312b01..e3ef8e25e 100644 --- a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go +++ b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go @@ -99,9 +99,14 @@ func (c *fileStore) Store(authConfig types.AuthConfig) error { return nil } -// ConvertToHostname converts a registry url which has http|https prepended -// to just an hostname. -// Copied from github.com/docker/docker/registry.ConvertToHostname to reduce dependencies. +// ConvertToHostname normalizes a registry URL which has http|https prepended +// to just its hostname. It is used to match credentials, which may be either +// stored as hostname or as hostname including scheme (in legacy configuration +// files). +// +// It's the equivalent to [registry.ConvertToHostname] in the daemon. +// +// [registry.ConvertToHostname]: https://pkg.go.dev/github.com/moby/moby/v2@v2.0.0-beta.7/daemon/pkg/registry#ConvertToHostname func ConvertToHostname(maybeURL string) string { stripped := maybeURL if strings.Contains(stripped, "://") { diff --git a/vendor/github.com/docker/cli/cli/config/memorystore/store.go b/vendor/github.com/docker/cli/cli/config/memorystore/store.go index f8ec62b95..44523d392 100644 --- a/vendor/github.com/docker/cli/cli/config/memorystore/store.go +++ b/vendor/github.com/docker/cli/cli/config/memorystore/store.go @@ -1,5 +1,5 @@ // FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: -//go:build go1.24 +//go:build go1.25 package memorystore diff --git a/vendor/github.com/docker/distribution/LICENSE b/vendor/github.com/docker/distribution/LICENSE deleted file mode 100644 index e06d20818..000000000 --- a/vendor/github.com/docker/distribution/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go b/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go deleted file mode 100644 index 2c3ebe165..000000000 --- a/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go +++ /dev/null @@ -1,27 +0,0 @@ -package challenge - -import ( - "net/url" - "strings" -) - -// FROM: https://golang.org/src/net/http/http.go -// Given a string of the form "host", "host:port", or "[ipv6::address]:port", -// return true if the string includes a port. -func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } - -// FROM: http://golang.org/src/net/http/transport.go -var portMap = map[string]string{ - "http": "80", - "https": "443", -} - -// canonicalAddr returns url.Host but always with a ":port" suffix -// FROM: http://golang.org/src/net/http/transport.go -func canonicalAddr(url *url.URL) string { - addr := url.Host - if !hasPort(addr) { - return addr + ":" + portMap[url.Scheme] - } - return addr -} diff --git a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go b/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go deleted file mode 100644 index 69021bcee..000000000 --- a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package estargz adapts the containerd estargz package to our abstractions. -package estargz - -import ( - "bytes" - "io" - - "github.com/containerd/stargz-snapshotter/estargz" - v1 "github.com/google/go-containerregistry/pkg/v1" -) - -// Assert that what we're returning is an io.ReadCloser -var _ io.ReadCloser = (*estargz.Blob)(nil) - -// ReadCloser reads uncompressed tarball input from the io.ReadCloser and -// returns: -// - An io.ReadCloser from which compressed data may be read, and -// - A v1.Hash with the hash of the estargz table of contents, or -// - An error if the estargz processing encountered a problem. -// -// Refer to estargz for the options: -// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option -func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) { - defer r.Close() - - // TODO(#876): Avoid buffering into memory. - bs, err := io.ReadAll(r) - if err != nil { - return nil, v1.Hash{}, err - } - br := bytes.NewReader(bs) - - rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...) - if err != nil { - return nil, v1.Hash{}, err - } - - h, err := v1.NewHash(rc.TOCDigest().String()) - return rc, h, err -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index 6f08460ba..ade2c63c9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -25,7 +25,6 @@ import ( "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/config/types" "github.com/google/go-containerregistry/pkg/name" - "github.com/mitchellh/go-homedir" ) // Resource represents a registry or repository that can be authenticated against. @@ -95,7 +94,7 @@ func (dk *defaultKeychain) ResolveContext(_ context.Context, target Resource) (A // First, check $HOME/.docker/config.json foundDockerConfig := false - home, err := homedir.Dir() + home, err := os.UserHomeDir() if err == nil { foundDockerConfig = fileExists(filepath.Join(home, ".docker/config.json")) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index 5e6b6e62a..7531d2426 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -24,8 +24,8 @@ import ( "strings" ) -// Detect more complex forms of local references. -var reLocal = regexp.MustCompile(`.*\.local(?:host)?(?::\d{1,5})?$`) +// Detect more complex forms of localhost references. +var reLocal = regexp.MustCompile(`.*\.localhost(?::\d{1,5})?$`) // Detect the loopback IP (127.0.0.1) var reLoopback = regexp.MustCompile(regexp.QuoteMeta("127.0.0.1")) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go index 22d483f3b..783eecc69 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go @@ -29,6 +29,7 @@ type Manifest struct { Layers []Descriptor `json:"layers"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // IndexManifest represents an OCI image index in a structured way. @@ -38,6 +39,7 @@ type IndexManifest struct { Manifests []Descriptor `json:"manifests"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // Descriptor holds a reference from the manifest to one of its constituent elements. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go index 3ea27fe47..0c58e0c1b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go @@ -52,6 +52,16 @@ func (i *image) MediaType() (types.MediaType, error) { return i.base.MediaType() } +// isImageConfig reports whether the media type is a Docker or OCI image config. +func isImageConfig(mt types.MediaType) bool { + switch mt { + case types.DockerConfigJSON, types.OCIConfigJSON: + return true + default: + return false + } +} + func (i *image) compute() error { i.Lock() defer i.Unlock() @@ -60,6 +70,24 @@ func (i *image) compute() error { if i.computed { return nil } + + m, err := i.base.Manifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + + diffIDMap := make(map[v1.Hash]v1.Layer) + digestMap := make(map[v1.Hash]v1.Layer) + + // Determine effective config media type (user override takes precedence). + cfgMediaType := manifest.Config.MediaType + if i.configMediaType != nil { + cfgMediaType = *i.configMediaType + } + + imageConfig := isImageConfig(cfgMediaType) + var configFile *v1.ConfigFile if i.configFile != nil { configFile = i.configFile @@ -70,33 +98,32 @@ func (i *image) compute() error { } configFile = cf.DeepCopy() } - diffIDs := configFile.RootFS.DiffIDs - history := configFile.History - diffIDMap := make(map[v1.Hash]v1.Layer) - digestMap := make(map[v1.Hash]v1.Layer) + // For image configs, update RootFS.DiffIDs and History from added layers. + // For artifacts, skip this: the config has no rootfs or history fields. + if imageConfig { + diffIDs := configFile.RootFS.DiffIDs + history := configFile.History - for _, add := range i.adds { - history = append(history, add.History) - if add.Layer != nil { - diffID, err := add.Layer.DiffID() - if err != nil { - return err + for _, add := range i.adds { + history = append(history, add.History) + if add.Layer != nil { + diffID, err := add.Layer.DiffID() + if err != nil { + return err + } + diffIDs = append(diffIDs, diffID) + diffIDMap[diffID] = add.Layer } - diffIDs = append(diffIDs, diffID) - diffIDMap[diffID] = add.Layer } - } - m, err := i.base.Manifest() - if err != nil { - return err + configFile.RootFS.DiffIDs = diffIDs + configFile.History = history } - manifest := m.DeepCopy() + manifestLayers := manifest.Layers for _, add := range i.adds { if add.Layer == nil { - // Empty layers include only history in manifest. continue } @@ -105,14 +132,12 @@ func (i *image) compute() error { return err } - // Fields in the addendum override the original descriptor. if len(add.Annotations) != 0 { desc.Annotations = add.Annotations } if len(add.URLs) != 0 { desc.URLs = add.URLs } - if add.MediaType != "" { desc.MediaType = add.MediaType } @@ -120,42 +145,38 @@ func (i *image) compute() error { manifestLayers = append(manifestLayers, *desc) digestMap[desc.Digest] = add.Layer } - - configFile.RootFS.DiffIDs = diffIDs - configFile.History = history - manifest.Layers = manifestLayers - rcfg, err := json.Marshal(configFile) - if err != nil { - return err - } - d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) - if err != nil { - return err - } - manifest.Config.Digest = d - manifest.Config.Size = sz + // For image configs, re-marshal the config and update the manifest digest. + // For artifacts, preserve the original config blob as-is to avoid + // corrupting the digest via re-marshaling. + if imageConfig { + rcfg, err := json.Marshal(configFile) + if err != nil { + return err + } + d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) + if err != nil { + return err + } + manifest.Config.Digest = d + manifest.Config.Size = sz - // If Data was set in the base image, we need to update it in the mutated image. - if m.Config.Data != nil { - manifest.Config.Data = rcfg + if m.Config.Data != nil { + manifest.Config.Data = rcfg + } } - // If the user wants to mutate the media type of the config if i.configMediaType != nil { manifest.Config.MediaType = *i.configMediaType } - if i.mediaType != nil { manifest.MediaType = *i.mediaType } - if i.annotations != nil { if manifest.Annotations == nil { manifest.Annotations = map[string]string{} } - for k, v := range i.annotations { manifest.Annotations[k] = v } @@ -173,29 +194,34 @@ func (i *image) compute() error { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. func (i *image) Layers() ([]v1.Layer, error) { - if err := i.compute(); errors.Is(err, stream.ErrNotComputed) { - // Image contains a streamable layer which has not yet been - // consumed. Just return the layers we have in case the caller - // is going to consume the layers. + if err := i.compute(); errors.Is(err, stream.ErrNotComputed) || (i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType)) { + // Stream not yet consumed, or non-image OCI artifact (RootFS.DiffIDs + // is empty so partial.DiffIDs returns nothing). Fall back to the base + // layers plus any added layers. layers, err := i.base.Layers() if err != nil { return nil, err } for _, add := range i.adds { - layers = append(layers, add.Layer) + if add.Layer != nil { + layers = append(layers, add.Layer) + } } return layers, nil } else if err != nil { return nil, err } - diffIDs, err := partial.DiffIDs(i) - if err != nil { - return nil, err - } - ls := make([]v1.Layer, 0, len(diffIDs)) - for _, h := range diffIDs { - l, err := i.LayerByDiffID(h) + // Walk manifest layer descriptors by digest rather than rootfs diff + // IDs. Two layers can legitimately share a diff ID — same uncompressed + // content, different compression — and produce distinct digests. The + // manifest preserves the per-occurrence digest; LayerByDiffID does not, + // which previously caused duplicate-diff-ID layers to collapse to a + // single entry in the returned slice and break blob upload for + // downstream pushers (see #2034). + ls := make([]v1.Layer, 0, len(i.manifest.Layers)) + for _, desc := range i.manifest.Layers { + l, err := i.LayerByDigest(desc.Digest) if err != nil { return nil, err } @@ -220,11 +246,18 @@ func (i *image) ConfigFile() (*v1.ConfigFile, error) { return i.configFile.DeepCopy(), nil } -// RawConfigFile returns the serialized bytes of ConfigFile() +// RawConfigFile returns the serialized bytes of ConfigFile(). +// For non-image OCI artifacts, returns the original raw config to preserve +// the config blob digest. func (i *image) RawConfigFile() ([]byte, error) { if err := i.compute(); err != nil { return nil, err } + // If the manifest config is not a standard image config, return the + // original raw bytes to avoid corrupting the digest via re-marshaling. + if i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType) { + return i.base.RawConfigFile() + } return json.Marshal(i.configFile) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index bf1113c9f..a6186b375 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -277,67 +277,91 @@ func extract(img v1.Image, w io.Writer) error { // whiteout layers more efficient, since we can just keep track of the removed // files as we see .wh. layers and ignore those in previous layers. for i := len(layers) - 1; i >= 0; i-- { - layer := layers[i] - layerReader, err := layer.Uncompressed() + if err := extractLayer(tarWriter, fileMap, layers[i]); err != nil { + return err + } + } + return nil +} + +func extractLayer(tarWriter *tar.Writer, fileMap map[string]bool, layer v1.Layer) error { + layerReader, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("reading layer contents: %w", err) + } + defer layerReader.Close() + + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } if err != nil { - return fmt.Errorf("reading layer contents: %w", err) + return fmt.Errorf("reading tar: %w", err) } - defer layerReader.Close() - tarReader := tar.NewReader(layerReader) - for { - header, err := tarReader.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("reading tar: %w", err) - } - // Some tools prepend everything with "./", so if we don't Clean the - // name, we may have duplicate entries, which angers tar-split. - header.Name = filepath.Clean(header.Name) - // force PAX format to remove Name/Linkname length limit of 100 characters - // required by USTAR and to not depend on internal tar package guess which - // prefers USTAR over PAX - header.Format = tar.FormatPAX - - basename := filepath.Base(header.Name) - dirname := filepath.Dir(header.Name) - tombstone := strings.HasPrefix(basename, whiteoutPrefix) - if tombstone { - basename = basename[len(whiteoutPrefix):] + // Some tools prepend everything with "./", so if we don't Clean the + // name, we may have duplicate entries, which angers tar-split. + header.Name = filepath.Clean(header.Name) + + // Reject relative symlinks and hardlinks whose targets escape the + // image rootfs. Relative targets are resolved against the symlink's + // own directory: if the clean result starts with ".." the link would + // leave the rootfs. Relative symlinks that stay within the rootfs + // (common for glibc, C toolchains, etc.) are preserved unchanged. + // Absolute targets are left as-is; see #2238 for ongoing discussion + // on whether they should be pruned. + if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink { + if !filepath.IsAbs(header.Linkname) { + resolved := filepath.Clean(filepath.Join(filepath.Dir(header.Name), header.Linkname)) //nolint:gosec // G305: path is only used for validation, not file I/O + if strings.HasPrefix(resolved, "..") { + continue + } } + } - // check if we have seen value before - // if we're checking a directory, don't filepath.Join names - var name string - if header.Typeflag == tar.TypeDir { - name = header.Name - } else { - name = filepath.Join(dirname, basename) - } + // force PAX format to remove Name/Linkname length limit of 100 characters + // required by USTAR and to not depend on internal tar package guess which + // prefers USTAR over PAX + header.Format = tar.FormatPAX - if _, ok := fileMap[name]; ok && !tombstone { - continue - } + basename := filepath.Base(header.Name) + dirname := filepath.Dir(header.Name) + tombstone := strings.HasPrefix(basename, whiteoutPrefix) + if tombstone { + basename = basename[len(whiteoutPrefix):] + } - // check for a whited out parent directory - if inWhiteoutDir(fileMap, name) { - continue - } + // check if we have seen value before + // if we're checking a directory, don't filepath.Join names + var name string + if header.Typeflag == tar.TypeDir { + name = header.Name + } else { + name = filepath.Join(dirname, basename) + } + + if _, ok := fileMap[name]; ok && !tombstone { + continue + } - // mark file as handled. non-directory implicitly tombstones - // any entries with a matching (or child) name - fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) - if !tombstone { - if err := tarWriter.WriteHeader(header); err != nil { + // check for a whited out parent directory + if inWhiteoutDir(fileMap, name) { + continue + } + + // mark file as handled. non-directory implicitly tombstones + // any entries with a matching (or child) name + fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) + if !tombstone { + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if header.Size > 0 { + if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { return err } - if header.Size > 0 { - if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { - return err - } - } } } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index c8b22b3f9..e83b4a882 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -337,8 +337,12 @@ func Descriptor(d Describable) (*v1.Descriptor, error) { mf, _ := Manifest(wrm) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - desc.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) + } } } } @@ -429,7 +433,10 @@ func ArtifactType(w WithManifest) (string, error) { mf, _ := w.Manifest() // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { + if mf != nil { + if mf.ArtifactType != "" { + return mf.ArtifactType, nil + } return string(mf.Config.MediaType), nil } return "", nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go index d77b37c0c..3bda1d09c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -40,8 +41,9 @@ const ( // fetcher implements methods for reading from a registry. type fetcher struct { - target resource - client *http.Client + target resource + client *http.Client + limiter *pullLimiter } func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) { @@ -69,10 +71,40 @@ func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, er } return &fetcher{ target: target, - client: &http.Client{Transport: tr}, + client: &http.Client{ + Transport: tr, + CheckRedirect: checkRedirectSSRF, + }, + limiter: o.limiter, }, nil } +// checkRedirectSSRF rejects HTTP redirects that cross from a public host to a +// private or link-local IP literal. This prevents a malicious registry from +// issuing a 302 to a cloud instance metadata service (e.g. 169.254.169.254) +// or another internal network address during blob or manifest downloads. +// +// Same-host redirects and redirects to non-IP hostnames (including DNS names +// that may resolve to private addresses) are allowed. The first redirect in +// the chain uses the original request URL as the "origin host" via +// req.Response.Request, falling back to req.URL when no prior response exists. +func checkRedirectSSRF(req *http.Request, via []*http.Request) error { + if len(via) == 0 || req.Response == nil { + return nil + } + origHost := via[0].URL.Hostname() + destHost := req.URL.Hostname() + if destHost == origHost { + return nil // same-host redirect is always allowed + } + if ip := net.ParseIP(destHost); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return fmt.Errorf("SSRF protection: redirect from %q to private/link-local host %q denied", origHost, destHost) + } + } + return nil +} + func (f *fetcher) Do(req *http.Request) (*http.Response, error) { return f.client.Do(req) } @@ -162,11 +194,20 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta } var artifactType string + var annotations map[string]string mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - artifactType = string(mf.Config.MediaType) + if mf != nil { + // Per the OCI distribution spec, artifactType on the descriptor is + // set to the manifest's artifactType if present, otherwise it falls + // back to the config descriptor's mediaType. + if mf.ArtifactType != "" { + artifactType = mf.ArtifactType + } else { + artifactType = string(mf.Config.MediaType) + } + annotations = mf.Annotations } // Do nothing for tags; I give up. @@ -183,6 +224,7 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta Size: size, MediaType: mediaType, ArtifactType: artifactType, + Annotations: annotations, } return manifest, &desc, nil @@ -247,18 +289,30 @@ func (f *fetcher) headManifest(ctx context.Context, ref name.Reference, acceptab func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) { u := f.url("blobs", h.String()) + return f.fetchBlobURL(ctx, u, size, h) +} + +func (f *fetcher) fetchBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash) (io.ReadCloser, error) { + release, err := f.limiter.acquire(ctx) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { + release() return nil, err } resp, err := f.client.Do(req.WithContext(ctx)) if err != nil { + release() return nil, redact.Error(err) } if err := transport.CheckError(resp, http.StatusOK); err != nil { resp.Body.Close() + release() return nil, err } @@ -269,11 +323,22 @@ func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.Read if size == verify.SizeUnknown { size = hsize } else if hsize != size { + resp.Body.Close() + release() return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size) } } - return verify.ReadCloser(resp.Body, size, h) + rc, err := verify.ReadCloser(resp.Body, size, h) + if err != nil { + resp.Body.Close() + release() + return nil, err + } + return &limitedReadCloser{ + ReadCloser: rc, + release: release, + }, nil } func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, error) { @@ -296,6 +361,57 @@ func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, erro return resp, nil } +// validateForeignURL rejects foreign layer URLs that use a disallowed scheme +// or resolve to a private / link-local IP address (SSRF protection). DNS-based +// SSRF is out of scope, matching transport.validateRealmURL. +func validateForeignURL(rawURL string, insecure bool) error { + u, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("parsing foreign layer URL %q: %w", rawURL, err) + } + switch u.Scheme { + case "https": + case "http": + if !insecure { + return fmt.Errorf("foreign layer URL scheme %q not allowed for a secure registry; use https", u.Scheme) + } + default: + return fmt.Errorf("foreign layer URL scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) + } + host := u.Hostname() + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return fmt.Errorf("foreign layer URL host %q is a private or link-local address", host) + } + } + return nil +} + +// fetchForeignBlobURL fetches a foreign-layer blob, validating every redirect +// destination through validateForeignURL (SSRF protection). +func (f *fetcher) fetchForeignBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash, insecure bool) (io.ReadCloser, error) { + safeClient := &http.Client{ + Transport: f.client.Transport, + CheckRedirect: func(req *http.Request, _ []*http.Request) error { + return validateForeignURL(req.URL.String(), insecure) + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := safeClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: unexpected status %s", u.String(), resp.Status) + } + return verify.ReadCloser(resp.Body, size, h) +} + func (f *fetcher) blobExists(ctx context.Context, h v1.Hash) (bool, error) { u := f.url("blobs", h.String()) req, err := http.NewRequest(http.MethodHead, u.String(), nil) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index f085967ed..d39a0e71f 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "io" - "net/http" "net/url" "sync" @@ -27,7 +26,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -172,7 +170,7 @@ func (rl *remoteImageLayer) Digest() (v1.Hash, error) { // Compressed implements partial.CompressedLayer func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { - urls := []url.URL{rl.ri.fetcher.url("blobs", rl.digest.String())} + u := rl.ri.fetcher.url("blobs", rl.digest.String()) // Add alternative layer sources from URLs (usually none). d, err := partial.BlobDescriptor(rl, rl.digest) @@ -187,38 +185,38 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { // We don't want to log binary layers -- this can break terminals. ctx := redact.NewContext(rl.ctx, "omitting binary blobs from logs") - for _, s := range d.URLs { - u, err := url.Parse(s) - if err != nil { - return nil, err - } - urls = append(urls, *u) - } + insecure := rl.ri.fetcher.target.Scheme() == "http" // The lastErr for most pulls will be the same (the first error), but for // foreign layers we'll want to surface the last one, since we try to pull // from the registry first, which would often fail. // TODO: Maybe we don't want to try pulling from the registry first? var lastErr error - for _, u := range urls { - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { + rc, err := rl.ri.fetcher.fetchBlobURL(ctx, u, d.Size, rl.digest) + if err == nil { + return rc, nil + } + lastErr = err + + foreignURLs := make([]url.URL, 0, len(d.URLs)) + for _, s := range d.URLs { + if err := validateForeignURL(s, insecure); err != nil { return nil, err } - - resp, err := rl.ri.fetcher.Do(req.WithContext(ctx)) + fu, err := url.Parse(s) if err != nil { - lastErr = err - continue + return nil, err } + foreignURLs = append(foreignURLs, *fu) + } - if err := transport.CheckError(resp, http.StatusOK); err != nil { - resp.Body.Close() + for _, fu := range foreignURLs { + rc, err := rl.ri.fetcher.fetchForeignBlobURL(ctx, fu, d.Size, rl.digest, insecure) + if err != nil { lastErr = err continue } - - return verify.ReadCloser(resp.Body, d.Size, rl.digest) + return rc, nil } return nil, lastErr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index b80972c80..3956a38e6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -225,8 +225,12 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - child.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + child.ArtifactType = mf.ArtifactType + } else { + child.ArtifactType = string(mf.Config.MediaType) + } } } diff --git a/vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge/authchallenge.go similarity index 65% rename from vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go rename to vendor/github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge/authchallenge.go index fe238210c..9084377c2 100644 --- a/vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge/authchallenge.go @@ -1,91 +1,26 @@ -package challenge +// Copyright 2014 Docker, Inc. +// Copyright 2021-2026 The Distribution contributors +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authchallenge import ( - "fmt" "net/http" - "net/url" "strings" - "sync" ) -// Challenge carries information from a WWW-Authenticate response header. -// See RFC 2617. -type Challenge struct { - // Scheme is the auth-scheme according to RFC 2617 - Scheme string - - // Parameters are the auth-params according to RFC 2617 - Parameters map[string]string -} - -// Manager manages the challenges for endpoints. -// The challenges are pulled out of HTTP responses. Only -// responses which expect challenges should be added to -// the manager, since a non-unauthorized request will be -// viewed as not requiring challenges. -type Manager interface { - // GetChallenges returns the challenges for the given - // endpoint URL. - GetChallenges(endpoint url.URL) ([]Challenge, error) - - // AddResponse adds the response to the challenge - // manager. The challenges will be parsed out of - // the WWW-Authenicate headers and added to the - // URL which was produced the response. If the - // response was authorized, any challenges for the - // endpoint will be cleared. - AddResponse(resp *http.Response) error -} - -// NewSimpleManager returns an instance of -// Manger which only maps endpoints to challenges -// based on the responses which have been added the -// manager. The simple manager will make no attempt to -// perform requests on the endpoints or cache the responses -// to a backend. -func NewSimpleManager() Manager { - return &simpleManager{ - Challenges: make(map[string][]Challenge), - } -} - -type simpleManager struct { - sync.RWMutex - Challenges map[string][]Challenge -} - -func normalizeURL(endpoint *url.URL) { - endpoint.Host = strings.ToLower(endpoint.Host) - endpoint.Host = canonicalAddr(endpoint) -} - -func (m *simpleManager) GetChallenges(endpoint url.URL) ([]Challenge, error) { - normalizeURL(&endpoint) - - m.RLock() - defer m.RUnlock() - challenges := m.Challenges[endpoint.String()] - return challenges, nil -} - -func (m *simpleManager) AddResponse(resp *http.Response) error { - challenges := ResponseChallenges(resp) - if resp.Request == nil { - return fmt.Errorf("missing request reference") - } - urlCopy := url.URL{ - Path: resp.Request.URL.Path, - Host: resp.Request.URL.Host, - Scheme: resp.Request.URL.Scheme, - } - normalizeURL(&urlCopy) - - m.Lock() - defer m.Unlock() - m.Challenges[urlCopy.String()] = challenges - return nil -} - // Octet types from RFC 2616. type octetType byte @@ -113,7 +48,7 @@ func init() { // token = 1* // qdtext = > - for c := 0; c < 256; c++ { + for c := range 256 { var t octetType isCtl := c <= 31 || c == 127 isChar := 0 <= c && c <= 127 @@ -128,6 +63,16 @@ func init() { } } +// Challenge carries information from a WWW-Authenticate response header. +// See RFC 2617. +type Challenge struct { + // Scheme is the auth-scheme according to RFC 2617 + Scheme string + + // Parameters are the auth-params according to RFC 2617 + Parameters map[string]string +} + // ResponseChallenges returns a list of authorization challenges // for the given http Response. Challenges are only checked if // the response status code was a 401. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go new file mode 100644 index 000000000..54ec3b28d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go @@ -0,0 +1,55 @@ +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "io" + "sync" +) + +type pullLimiter struct { + tokens chan struct{} +} + +func newPullLimiter(jobs int) *pullLimiter { + return &pullLimiter{ + tokens: make(chan struct{}, jobs), + } +} + +func (l *pullLimiter) acquire(ctx context.Context) (func(), error) { + if l == nil { + return func() {}, nil + } + select { + case l.tokens <- struct{}{}: + return func() { <-l.tokens }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +type limitedReadCloser struct { + io.ReadCloser + release func() + once sync.Once +} + +func (l *limitedReadCloser) Close() error { + err := l.ReadCloser.Close() + l.once.Do(l.release) + return err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 15b7da1e4..3d8bb8ddf 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -45,6 +45,7 @@ type options struct { retryBackoff Backoff retryPredicate retry.Predicate retryStatusCodes []int + limiter *pullLimiter // Only these options can overwrite Reuse()d options. platform v1.Platform @@ -92,6 +93,7 @@ var fastBackoff = Backoff{ var defaultRetryStatusCodes = []int{ http.StatusRequestTimeout, + http.StatusTooManyRequests, // 429: OCI distribution-spec rate limit; TooManyRequestsErrorCode is already classified temporary in transport/error.go http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, @@ -142,6 +144,7 @@ func makeOptions(opts ...Option) (*options, error) { return nil, err } } + o.limiter = newPullLimiter(o.jobs) switch { case o.auth != nil && o.keychain != nil: diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go index 47e3b806e..5675c97c8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go @@ -156,7 +156,11 @@ func (p *Pusher) Upload(ctx context.Context, repo name.Repository, l v1.Layer) e } func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { - w, err := p.writer(ctx, ref.Context(), p.o) + // Use a transport scoped for delete. Requesting DeleteScope (which + // includes the "delete" action) allows registries that require an + // explicit delete permission—such as IBM Cloud Container Registry—to + // grant access. + client, err := makeDeleteClient(ctx, ref.Context(), p.o) if err != nil { return err } @@ -172,7 +176,7 @@ func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { return err } - resp, err := w.w.client.Do(req.WithContext(ctx)) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go index 4bc6f70a8..17e9c26ed 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go @@ -67,7 +67,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, var b []byte if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Type") == string(types.OCIImageIndex) { - b, err = io.ReadAll(resp.Body) + b, err = io.ReadAll(io.LimitReader(resp.Body, manifestLimit)) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index ea652d4ae..f845dd1fb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -26,14 +26,17 @@ import ( "strings" "sync" - authchallenge "github.com/docker/distribution/registry/client/auth/challenge" - "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge" ) +// maxTokenBodySize limits bearer token response body reads to prevent OOM +// when a token endpoint returns an unexpectedly large body. +const maxTokenBodySize = 64 * 1024 // 64 KiB + type Token struct { Token string `json:"token"` AccessToken string `json:"access_token,omitempty"` @@ -83,6 +86,13 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip if !ok { return nil, fmt.Errorf("malformed www-authenticate, missing realm: %v", pr.Parameters) } + // Validate the realm URL before storing it. A malicious or compromised + // registry can supply a realm pointing at an internal service or cloud + // metadata endpoint (e.g. 169.254.169.254), causing SSRF when the client + // subsequently fetches a token. + if err := validateRealmURL(realm, reg.RegistryStr(), pr.Insecure); err != nil { + return nil, fmt.Errorf("invalid realm in www-authenticate: %w", err) + } service := pr.Parameters["service"] scheme := "https" if pr.Insecure { @@ -99,6 +109,56 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip }, nil } +// realmRedirectCheck mimics the default http.Client redirect policy but also +// validates each redirect URL with validateRealmURL. +func realmRedirectCheck(registryHost string, insecure bool) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + if err := validateRealmURL(req.URL.String(), registryHost, insecure); err != nil { + return fmt.Errorf("refusing token-server redirect to %q: %w", req.URL, err) + } + return nil + } +} + +// validateRealmURL returns an error if the realm URL uses a disallowed scheme +// or resolves to a private / link-local IP address. Realm URLs matching the +// registry host:port are always allowed. See #2258. +func validateRealmURL(realm, registryHost string, insecure bool) error { + u, err := url.Parse(realm) + if err != nil { + return fmt.Errorf("parsing realm %q: %w", realm, err) + } + switch u.Scheme { + case "https": + // always allowed + case "http": + if !insecure { + return fmt.Errorf("realm scheme %q not allowed for a secure registry; use https", u.Scheme) + } + default: + return fmt.Errorf("realm scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) + } + // Always allow realms matching the registry host:port. + if registryHost != "" && u.Host == registryHost { + return nil + } + // Reject IP literals that resolve to private or link-local ranges. + // This blocks direct references to RFC 1918 addresses, loopback, and + // link-local ranges including the cloud instance metadata service + // (169.254.169.254 / fd00:ec2::254). DNS-based SSRF is out of scope + // here; callers should apply network-level controls if needed. + host := u.Hostname() + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return fmt.Errorf("realm host %q is a private or link-local address", host) + } + } + return nil +} + type bearerTransport struct { mx sync.RWMutex // Wrapped by bearerTransport. @@ -336,7 +396,8 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { v.Set("access_type", "offline") } - client := http.Client{Transport: bt.inner} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: bt.inner, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return nil, err @@ -359,7 +420,7 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return io.ReadAll(io.LimitReader(resp.Body, maxTokenBodySize)) } // https://docs.docker.com/registry/spec/auth/token/ @@ -373,7 +434,8 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { auth: bt.basic, target: u.Host, } - client := http.Client{Transport: b} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: b, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} v := u.Query() bt.mx.RLock() @@ -403,5 +465,5 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return io.ReadAll(io.LimitReader(resp.Body, maxTokenBodySize)) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index d38e67624..934484db8 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -15,6 +15,7 @@ package transport import ( + "bytes" "encoding/json" "fmt" "io" @@ -112,6 +113,10 @@ type ErrorCode string // The set of error conditions a registry may return: // https://github.com/distribution/distribution/blob/aac2f6c8b7c5a6c60190848bab5cbeed2b5ba0a9/docs/spec/api.md#errors-2 +// maxErrorBodySize limits HTTP error response body reads to prevent OOM when +// a registry returns an unexpectedly large error body. +const maxErrorBodySize = 64 * 1024 // 64 KiB + const ( BlobUnknownErrorCode ErrorCode = "BLOB_UNKNOWN" BlobUploadInvalidErrorCode ErrorCode = "BLOB_UPLOAD_INVALID" @@ -146,6 +151,7 @@ var temporaryErrorCodes = map[ErrorCode]struct{}{ var temporaryStatusCodes = map[int]struct{}{ http.StatusRequestTimeout: {}, + http.StatusTooManyRequests: {}, // matches TooManyRequestsErrorCode in temporaryErrorCodes http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, @@ -161,7 +167,7 @@ func CheckError(resp *http.Response, codes ...int) error { } } - b, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) if err != nil { return err } @@ -185,11 +191,16 @@ func makeError(resp *http.Response, body []byte) *Error { } func retryError(resp *http.Response) error { - b, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) if err != nil { return err } + // Restore the body so that a subsequent CheckError call (after the + // retry loop exhausts its retries) can still read and parse the + // structured registry error from the response. + resp.Body = io.NopCloser(bytes.NewReader(b)) + rerr := makeError(resp, b) rerr.temporary = true return rerr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go index 325874399..6d5cba6d4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go @@ -24,9 +24,9 @@ import ( "strings" "time" - authchallenge "github.com/docker/distribution/registry/client/auth/challenge" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge" ) // 300ms is the default fallback period for go's DNS dialer but we could make this configurable. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go index c3b56f7a4..6ee7782b7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go @@ -18,7 +18,9 @@ package transport const ( PullScope string = "pull" PushScope string = "push,pull" - // For now DELETE is PUSH, which is the read/write ACL. - DeleteScope string = PushScope + // DeleteScope requests "delete" in addition to push/pull so that + // registries requiring an explicit delete action (e.g. IBM Cloud + // Container Registry) grant the necessary access. + DeleteScope string = "push,pull,delete" CatalogScope string = "catalog" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 58adfd8d3..4ff30a8cb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "sort" @@ -66,6 +67,25 @@ type writer struct { scopes []string } +// makeDeleteClient returns an HTTP client whose token includes the "delete" +// action so that registries requiring an explicit delete permission grant +// access for manifest deletion. +func makeDeleteClient(ctx context.Context, repo name.Repository, o *options) (*http.Client, error) { + auth := o.auth + if o.keychain != nil { + kauth, err := authn.Resolve(ctx, o.keychain, repo) + if err != nil { + return nil, err + } + auth = kauth + } + tr, err := transport.NewWithContext(ctx, repo.Registry, auth, o.transport, []string{repo.Scope(transport.DeleteScope)}) + if err != nil { + return nil, err + } + return &http.Client{Transport: tr}, nil +} + func makeWriter(ctx context.Context, repo name.Repository, ls []v1.Layer, o *options) (*writer, error) { auth := o.auth if o.keychain != nil { @@ -148,7 +168,29 @@ func (w *writer) nextLocation(resp *http.Response) (string, error) { // If the location header returned is just a url path, then fully qualify it. // We cannot simply call w.url, since there might be an embedded query string. - return resp.Request.URL.ResolveReference(u).String(), nil + resolved := resp.Request.URL.ResolveReference(u) + + // Reject Location headers that redirect to a DIFFERENT host that resolves to + // a private or link-local IP literal. A malicious or compromised registry can + // respond to a blob upload initiation (POST /v2/.../blobs/uploads/) with a + // crafted Location header pointing at an internal service, causing the client + // to send subsequent PATCH/PUT requests (including the layer data as the body) + // to that internal address. Pre-signed blob URLs from cloud storage providers + // (GCS, S3, Azure Blob) use public hostnames, so legitimate cross-host + // redirects are unaffected. + // + // Same-host redirects (e.g. a different path on the registry itself) are + // always allowed regardless of whether the registry IP is private. + origHost := resp.Request.URL.Hostname() + if destHost := resolved.Hostname(); destHost != origHost { + if ip := net.ParseIP(destHost); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return "", fmt.Errorf("SSRF protection: Location header redirects to private/link-local host %q", destHost) + } + } + } + + return resolved.String(), nil } // checkExistingBlob checks if a blob exists already in the repository by making a @@ -555,9 +597,10 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } var mf struct { - MediaType types.MediaType `json:"mediaType"` - Subject *v1.Descriptor `json:"subject,omitempty"` - Config struct { + MediaType types.MediaType `json:"mediaType"` + Subject *v1.Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Config struct { MediaType types.MediaType `json:"mediaType"` } `json:"config"` } @@ -599,10 +642,14 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } desc := v1.Descriptor{ - ArtifactType: string(mf.Config.MediaType), - MediaType: mf.MediaType, - Digest: h, - Size: size, + MediaType: mf.MediaType, + Digest: h, + Size: size, + } + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) } if err := w.commitSubjectReferrers(ctx, ref.Context().Digest(mf.Subject.Digest.String()), diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index aba609dea..96c0cce49 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -219,6 +219,15 @@ type tarFile struct { } func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { + return followLinks(opener, filePath, make(map[string]bool)) +} + +func followLinks(opener Opener, filePath string, visited map[string]bool) (io.ReadCloser, error) { + if visited[filePath] { + return nil, fmt.Errorf("link cycle detected for %s", filePath) + } + visited[filePath] = true + f, err := opener() if err != nil { return nil, err @@ -242,7 +251,7 @@ func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { if hdr.Name == filePath { if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink { currentDir := filepath.Dir(filePath) - return extractFileFromTar(opener, path.Join(currentDir, path.Clean(hdr.Linkname))) + return followLinks(opener, path.Join(currentDir, path.Clean(hdr.Linkname)), visited) } needClose = false return tarFile{ diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 8a2630961..5e63d52d5 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -22,10 +22,7 @@ import ( "os" "sync" - "github.com/containerd/stargz-snapshotter/estargz" - "github.com/google/go-containerregistry/internal/and" comp "github.com/google/go-containerregistry/internal/compression" - gestargz "github.com/google/go-containerregistry/internal/estargz" ggzip "github.com/google/go-containerregistry/internal/gzip" "github.com/google/go-containerregistry/internal/zstd" "github.com/google/go-containerregistry/pkg/compression" @@ -43,7 +40,6 @@ type layer struct { compression compression.Compression compressionLevel int annotations map[string]string - estgzopts []estargz.Option mediaType types.MediaType } @@ -161,53 +157,15 @@ func WithCompressedCaching(l *layer) { // through estargz.Options to the underlying compression layer. This is // only meaningful when estargz is enabled. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargzOptions(opts ...estargz.Option) LayerOption { - return func(l *layer) { - l.estgzopts = opts - } +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargzOptions(...any) LayerOption { + return func(*layer) {} } // WithEstargz is a functional option that explicitly enables estargz support. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargz(l *layer) { - oguncompressed := l.uncompressedopener - estargz := func() (io.ReadCloser, error) { - crc, err := oguncompressed() - if err != nil { - return nil, err - } - eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compressionLevel)) - rc, h, err := gestargz.ReadCloser(crc, eopts...) - if err != nil { - return nil, err - } - l.annotations[estargz.TOCJSONDigestAnnotation] = h.String() - return &and.ReadCloser{ - Reader: rc, - CloseFunc: func() error { - err := rc.Close() - if err != nil { - return err - } - // As an optimization, leverage the DiffID exposed by the estargz ReadCloser - l.diffID, err = v1.NewHash(rc.DiffID().String()) - return err - }, - }, nil - } - uncompressed := func() (io.ReadCloser, error) { - urc, err := estargz() - if err != nil { - return nil, err - } - return ggzip.UnzipReadCloser(urc) - } - - l.compressedopener = estargz - l.uncompressedopener = uncompressed -} +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargz(*layer) {} // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) { @@ -241,11 +199,6 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { mediaType: types.DockerLayer, } - if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" { - logs.Warn.Println("GGCR_EXPERIMENT_ESTARGZ is deprecated, and will be removed in a future release.") - opts = append([]LayerOption{WithEstargz}, opts...) - } - switch comp { case compression.GZip: layer.compressedopener = opener diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes index 402433593..57aa6487c 100644 --- a/vendor/github.com/klauspost/compress/.gitattributes +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -1,2 +1,3 @@ * -text *.bin -text -diff +*.md text eol=lf diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index 4528059ca..804a20181 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -31,6 +31,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2d" binary: s2d @@ -57,6 +60,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2sx" binary: s2sx @@ -84,6 +90,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm archives: - @@ -91,7 +100,7 @@ archives: name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" format_overrides: - goos: windows - format: zip + formats: ['zip'] files: - unpack/* - s2/LICENSE diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 5125c1f26..fb023f2cf 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -1,693 +1,700 @@ -# compress - -This package provides various compression algorithms. - -* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. -* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). -* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. -* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. -* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. -* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. - -[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) -[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) -[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) - -# package usage - -Use `go get github.com/klauspost/compress@latest` to add it to your project. - -This package will support the current Go version and 2 versions back. - -* Use the `nounsafe` tag to disable all use of the "unsafe" package. -* Use the `noasm` tag to disable all assembly across packages. - -Use the links above for more information on each. - -# changelog -* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) - * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). - -* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) - * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 - * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 - -* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED - * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 - * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 - * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 - * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 - * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 - * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 - * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 - * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 - -* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) - * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 - * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 - * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 - * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 - * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 - * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 - * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 - -
- See changes to v1.17.x - -* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) - * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 - * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 - * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 - * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 - -* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) - * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 - * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 - * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 - * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 - * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 - -* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) - * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 - * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 - * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 - * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 - -* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) - * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 - * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 - -* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) - * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 - * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 - -* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) - * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 - * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 - -* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) - * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 - * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 - * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 - * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 - * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 -https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 - -* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) - * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 - * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 - * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 - * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 - * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 - -* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) - * fse: Fix max header size https://github.com/klauspost/compress/pull/881 - * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 - * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 - -* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) - * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 - -* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) - * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 - * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 - * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 - -* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) - * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 - * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 - * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 - * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 - * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 - * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 - -
-
- See changes to v1.16.x - - -* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) - * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 - * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 - -* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) - * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 - * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 - * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 - * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 - -* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) - * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 - * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 - -* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) - * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 - * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 - * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 - * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 - * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 - * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 - -* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) - * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 - * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 - * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 - * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 - * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 - -* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) - * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 - * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 - * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 - * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 - * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 - * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 -
- -
- See changes to v1.15.x - -* Jan 21st, 2023 (v1.15.15) - * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 - * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 - * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 - * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 - -* Jan 3rd, 2023 (v1.15.14) - - * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 - * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 - * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 - * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 - -* Dec 11, 2022 (v1.15.13) - * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 - * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 - -* Oct 26, 2022 (v1.15.12) - - * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 - * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 - -* Sept 26, 2022 (v1.15.11) - - * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 - * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 - * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 - * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 - -* Sept 16, 2022 (v1.15.10) - - * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 - * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 - * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 - * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 - * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 - * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 - * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 - -* July 21, 2022 (v1.15.9) - - * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 - * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 - * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 - -* July 13, 2022 (v1.15.8) - - * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 - * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 - * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 - * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 - * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 - * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 - * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 - -* June 29, 2022 (v1.15.7) - - * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 - * zip: Merge upstream https://github.com/klauspost/compress/pull/631 - * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 - * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 - * flate: Faster histograms https://github.com/klauspost/compress/pull/620 - * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 - -* June 3, 2022 (v1.15.6) - * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 - * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 - * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 - * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 - * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 - * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 - * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 - * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 - * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 - * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 - -* May 25, 2022 (v1.15.5) - * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 - * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 - * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 - * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 - * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 - * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 - * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 - * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 - * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 - - -* May 11, 2022 (v1.15.4) - * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) - * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) - * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) - * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) - -* May 5, 2022 (v1.15.3) - * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) - * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) - -* Apr 26, 2022 (v1.15.2) - * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) - * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) - * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) - * Minimum version is Go 1.16, added CI test on 1.18. - -* Mar 11, 2022 (v1.15.1) - * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) - * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) - * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) - * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) - * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) - -* Mar 3, 2022 (v1.15.0) - * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) - * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) - * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) - * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) - * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) - * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) - -Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. - -Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. - -While the release has been extensively tested, it is recommended to testing when upgrading. - -
- -
- See changes to v1.14.x - -* Feb 22, 2022 (v1.14.4) - * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) - * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) - * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 - * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) - -* Feb 17, 2022 (v1.14.3) - * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) - * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) - * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) - -* Jan 25, 2022 (v1.14.2) - * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) - * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) - * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) - * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) - * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) - * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) - -* Jan 11, 2022 (v1.14.1) - * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) - * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) - * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) - * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) - * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) -
- -
- See changes to v1.13.x - -* Aug 30, 2021 (v1.13.5) - * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) - * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) - * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) - * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) - -* Aug 12, 2021 (v1.13.4) - * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). - * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) - -* Aug 3, 2021 (v1.13.3) - * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) - * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) - * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) - * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) - * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) - * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) - -* Jun 14, 2021 (v1.13.1) - * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) - * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) - * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) - * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) - -* Jun 3, 2021 (v1.13.0) - * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. - * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) - * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) -
- - -
- See changes to v1.12.x - -* May 25, 2021 (v1.12.3) - * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) - * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) - * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) - -* Apr 27, 2021 (v1.12.2) - * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) - * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) - * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) - * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) - * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) - * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) - -* Apr 14, 2021 (v1.12.1) - * snappy package removed. Upstream added as dependency. - * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) - * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) - * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) - * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) - * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) - * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) -
- -
- See changes to v1.11.x - -* Mar 26, 2021 (v1.11.13) - * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) - * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) - * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) - * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) - -* Mar 5, 2021 (v1.11.12) - * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). - * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) - -* Mar 1, 2021 (v1.11.9) - * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) - * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) - * s2: Fix binaries. - -* Feb 25, 2021 (v1.11.8) - * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. - * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) - * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) - * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) - * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) - -* Jan 14, 2021 (v1.11.7) - * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) - * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) - * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) - * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) - * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) - -* Jan 7, 2021 (v1.11.6) - * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) - * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) - -* Dec 20, 2020 (v1.11.4) - * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) - * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) - * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) - * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) - * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) - -* Nov 15, 2020 (v1.11.3) - * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) - * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) - -* Oct 11, 2020 (v1.11.2) - * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) - -* Oct 1, 2020 (v1.11.1) - * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) - -* Sept 8, 2020 (v1.11.0) - * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) - * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) - * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) -
- -
- See changes to v1.10.x - -* July 8, 2020 (v1.10.11) - * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) - * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) - -* June 23, 2020 (v1.10.10) - * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) - -* June 16, 2020 (v1.10.9): - * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) - * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) - * Fuzzit tests removed. The service has been purchased and is no longer available. - -* June 5, 2020 (v1.10.8): - * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) - -* June 1, 2020 (v1.10.7): - * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) - * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) - * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) - -* May 21, 2020: (v1.10.6) - * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) - * zstd: Stricter decompression checks. - -* April 12, 2020: (v1.10.5) - * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) - -* Apr 8, 2020: (v1.10.4) - * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) -* Mar 11, 2020: (v1.10.3) - * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) - * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) - * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) - * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) - * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) - -* Feb 27, 2020: (v1.10.2) - * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) - * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) - -* Feb 18, 2020: (v1.10.1) - * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) - * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) - * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) - -* Feb 4, 2020: (v1.10.0) - * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) - * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) - * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) - * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) - -
- -
- See changes prior to v1.10.0 - -* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). -* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) -* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. -* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. -* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) -* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. -* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) -* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features -* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) -* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) -* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. -* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) -* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) -* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) -* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. -* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. -* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) -* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. -* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) -* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. -* Nov 11, 2019: Reduce inflate memory use by 1KB. -* Nov 10, 2019: Less allocations in deflate bit writer. -* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. -* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) -* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) -* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) -* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) - -
- -
- See changes prior to v1.9.0 - -* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) -* Oct 3, 2019: Fix inconsistent results on broken zstd streams. -* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) -* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). -* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). -* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). -* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. -* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. -* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. -* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. -* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. -* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. -* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. -* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) -* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) -* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) -* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) -* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. -* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. -* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. -* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. -* June 17, 2019: zstd decompression bugfix. -* June 17, 2019: fix 32 bit builds. -* June 17, 2019: Easier use in modules (less dependencies). -* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. -* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. -* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. -* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! -* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. -* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. -* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). -* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. -* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). -* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. -* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. -* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. -* May 28, 2017: Reduce allocations when resetting decoder. -* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. -* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). -* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. -* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. -* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. -* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. -* Mar 24, 2016: Small speedup for level 1-3. -* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. -* Feb 19, 2016: Handle small payloads faster in level 1-3. -* Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. -* Feb 14, 2016: Snappy: Merge upstream changes. -* Feb 14, 2016: Snappy: Fix aggressive skipping. -* Feb 14, 2016: Snappy: Update benchmark. -* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. -* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. -* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. -* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. -* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. -* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. -* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. -* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. -* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! -* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). -* Nov 20 2015: Small optimization to bit writer on 64 bit systems. -* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). -* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. -* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file -* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. - -
- -# deflate usage - -The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: - -Typical speed is about 2x of the standard library packages. - -| old import | new import | Documentation | -|------------------|---------------------------------------|-------------------------------------------------------------------------| -| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | -| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | -| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | -| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | - -You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. - -The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). - -Currently there is only minor speedup on decompression (mostly CRC32 calculation). - -Memory usage is typically 1MB for a Writer. stdlib is in the same range. -If you expect to have a lot of concurrently allocated Writers consider using -the stateless compression described below. - -For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). - -To disable all assembly add `-tags=noasm`. This works across all packages. - -# Stateless compression - -This package offers stateless compression as a special option for gzip/deflate. -It will do compression but without maintaining any state between Write calls. - -This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. - -This is only relevant in cases where you expect to run many thousands of compressors concurrently, -but with very little activity. This is *not* intended for regular web servers serving individual requests. - -Because of this, the size of actual Write calls will affect output size. - -In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. - -For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) - -A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: - -```go - // replace 'ioutil.Discard' with your output. - gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) - if err != nil { - return err - } - defer gzw.Close() - - w := bufio.NewWriterSize(gzw, 4096) - defer w.Flush() - - // Write to 'w' -``` - -This will only use up to 4KB in memory when the writer is idle. - -Compression is almost always worse than the fastest compression level -and each write will allocate (a little) memory. - - -# Other packages - -Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): - -* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. -* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. -* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. -* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. -* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. -* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. -* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. - -# license - -This code is licensed under the same conditions as the original Go code. See LICENSE file. - - - - +# compress + +This package provides various compression algorithms. + +* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. +* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. +* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). +* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. +* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. + +[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) +[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) +[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) + +# package usage + +Use `go get github.com/klauspost/compress@latest` to add it to your project. + +This package will support the current Go version and 2 versions back. + +* Use the `nounsafe` tag to disable all use of the "unsafe" package. +* Use the `noasm` tag to disable all assembly across packages. + +Use the links above for more information on each. + +# changelog + +* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) + * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 + * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 + * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 + +* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) + * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). + +* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) + * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 + * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 + +* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED + * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 + * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 + * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 + * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 + * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 + * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 + * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 + * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 + +* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) + * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 + * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 + * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 + * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 + * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 + * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 + * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 + +
+ See changes to v1.17.x + +* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) + * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 + * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 + * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 + * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 + +* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) + * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 + * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 + * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 + * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 + * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 + +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + +* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) + * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 + * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 + +* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) + * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 + * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 + * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 + * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 + * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 +https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 + +* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) + * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 + * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 + * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 + * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 + * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 + +* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) + * fse: Fix max header size https://github.com/klauspost/compress/pull/881 + * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 + * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 + +* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) + * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 + +* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) + * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 + * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 + * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 + +* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) + * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 + * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 + * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 + * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 + * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 + * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 + +
+
+ See changes to v1.16.x + + +* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) + * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 + * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 + +* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) + * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 + * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 + * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 + * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 + +* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) + * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 + * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 + +* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) + * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 + * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 + * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 + * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 + * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 + * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + +* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) + * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 + * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 + * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 + * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 + * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 + +* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) + * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 + * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 + * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 + * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 + * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 + * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 +
+ +
+ See changes to v1.15.x + +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + +* Sept 16, 2022 (v1.15.10) + + * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 + * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 + * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 + * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 + * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 + * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 + * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 + +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 + +* July 13, 2022 (v1.15.8) + + * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 + * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 + * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 + * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 + * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 + * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 + * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 + +* June 29, 2022 (v1.15.7) + + * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 + * zip: Merge upstream https://github.com/klauspost/compress/pull/631 + * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 + * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 + * flate: Faster histograms https://github.com/klauspost/compress/pull/620 + * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 + +* June 3, 2022 (v1.15.6) + * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 + * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 + * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 + * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 + * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 + * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 + * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 + * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 + * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 + * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 + +* May 25, 2022 (v1.15.5) + * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 + * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 + * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 + * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 + * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 + * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 + * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 + * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 + * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 + + +* May 11, 2022 (v1.15.4) + * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) + * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) + * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) + * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) + +* May 5, 2022 (v1.15.3) + * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) + * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) + +* Apr 26, 2022 (v1.15.2) + * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) + * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) + * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) + * Minimum version is Go 1.16, added CI test on 1.18. + +* Mar 11, 2022 (v1.15.1) + * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) + * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) + * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) + * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) + * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) + +* Mar 3, 2022 (v1.15.0) + * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) + * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) + * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) + * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) + * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) + * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) + +Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. + +Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. + +While the release has been extensively tested, it is recommended to testing when upgrading. + +
+ +
+ See changes to v1.14.x + +* Feb 22, 2022 (v1.14.4) + * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) + * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) + * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 + * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) + +* Feb 17, 2022 (v1.14.3) + * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) + * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) + * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) + +* Jan 25, 2022 (v1.14.2) + * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) + * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) + * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) + * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) + * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) + * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) + +* Jan 11, 2022 (v1.14.1) + * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) + * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) + * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) + * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) + * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) +
+ +
+ See changes to v1.13.x + +* Aug 30, 2021 (v1.13.5) + * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) + * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) + * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) + * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) + +* Aug 12, 2021 (v1.13.4) + * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). + * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) + +* Aug 3, 2021 (v1.13.3) + * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) + * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) + * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) + * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) + * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) + * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) + +* Jun 14, 2021 (v1.13.1) + * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) + * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) + * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) + * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) + +* Jun 3, 2021 (v1.13.0) + * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. + * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) + * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) +
+ + +
+ See changes to v1.12.x + +* May 25, 2021 (v1.12.3) + * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) + * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) + * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) + +* Apr 27, 2021 (v1.12.2) + * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) + * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) + * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) + * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) + * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) + * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) + +* Apr 14, 2021 (v1.12.1) + * snappy package removed. Upstream added as dependency. + * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) + * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) + * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) + * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) + * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) + * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) +
+ +
+ See changes to v1.11.x + +* Mar 26, 2021 (v1.11.13) + * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) + * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) + * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) + * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) + +* Mar 5, 2021 (v1.11.12) + * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). + * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) + +* Mar 1, 2021 (v1.11.9) + * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) + * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) + * s2: Fix binaries. + +* Feb 25, 2021 (v1.11.8) + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. + * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) + * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) + * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) + * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) + +* Jan 14, 2021 (v1.11.7) + * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) + * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) + * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) + * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) + * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) + +* Jan 7, 2021 (v1.11.6) + * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) + * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) + +* Dec 20, 2020 (v1.11.4) + * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) + * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) + * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) + * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) + * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) + +* Nov 15, 2020 (v1.11.3) + * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) + * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) + +* Oct 11, 2020 (v1.11.2) + * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) + +* Oct 1, 2020 (v1.11.1) + * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) + +* Sept 8, 2020 (v1.11.0) + * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) + * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) + * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) +
+ +
+ See changes to v1.10.x + +* July 8, 2020 (v1.10.11) + * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) + * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) + +* June 23, 2020 (v1.10.10) + * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) + +* June 16, 2020 (v1.10.9): + * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) + * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) + * Fuzzit tests removed. The service has been purchased and is no longer available. + +* June 5, 2020 (v1.10.8): + * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) + +* June 1, 2020 (v1.10.7): + * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) + * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) + * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) + +* May 21, 2020: (v1.10.6) + * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) + * zstd: Stricter decompression checks. + +* April 12, 2020: (v1.10.5) + * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) + +* Apr 8, 2020: (v1.10.4) + * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) +* Mar 11, 2020: (v1.10.3) + * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) + * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) + * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) + * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) + * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) + +* Feb 27, 2020: (v1.10.2) + * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) + * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) + +* Feb 18, 2020: (v1.10.1) + * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) + * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) + * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) + +* Feb 4, 2020: (v1.10.0) + * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) + * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) + * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) + * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) + +
+ +
+ See changes prior to v1.10.0 + +* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). +* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) +* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. +* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. +* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) +* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. +* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) +* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features +* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) +* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) +* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. +* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) +* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) +* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) +* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. +* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. +* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) +* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. +* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) +* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. +* Nov 11, 2019: Reduce inflate memory use by 1KB. +* Nov 10, 2019: Less allocations in deflate bit writer. +* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. +* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) +* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) +* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) +* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) + +
+ +
+ See changes prior to v1.9.0 + +* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) +* Oct 3, 2019: Fix inconsistent results on broken zstd streams. +* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) +* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). +* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). +* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). +* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. +* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. +* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. +* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. +* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. +* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. +* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. +* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) +* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) +* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) +* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) +* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. +* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. +* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. +* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. +* June 17, 2019: zstd decompression bugfix. +* June 17, 2019: fix 32 bit builds. +* June 17, 2019: Easier use in modules (less dependencies). +* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. +* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. +* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. +* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! +* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. +* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. +* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). +* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. +* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). +* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. +* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. +* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. +* May 28, 2017: Reduce allocations when resetting decoder. +* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. +* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). +* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. +* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. +* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. +* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. +* Mar 24, 2016: Small speedup for level 1-3. +* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. +* Feb 19, 2016: Handle small payloads faster in level 1-3. +* Feb 19, 2016: Added faster level 2 + 3 compression modes. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. +* Feb 14, 2016: Snappy: Merge upstream changes. +* Feb 14, 2016: Snappy: Fix aggressive skipping. +* Feb 14, 2016: Snappy: Update benchmark. +* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. +* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. +* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. +* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. +* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. +* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. +* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. +* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. +* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! +* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). +* Nov 20 2015: Small optimization to bit writer on 64 bit systems. +* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). +* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. +* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file +* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. + +
+ +# deflate usage + +The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: + +Typical speed is about 2x of the standard library packages. + +| old import | new import | Documentation | +|------------------|---------------------------------------|-------------------------------------------------------------------------| +| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | +| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | +| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | +| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | + +You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. + +The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). + +Currently there is only minor speedup on decompression (mostly CRC32 calculation). + +Memory usage is typically 1MB for a Writer. stdlib is in the same range. +If you expect to have a lot of concurrently allocated Writers consider using +the stateless compression described below. + +For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). + +To disable all assembly add `-tags=noasm`. This works across all packages. + +# Stateless compression + +This package offers stateless compression as a special option for gzip/deflate. +It will do compression but without maintaining any state between Write calls. + +This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. + +This is only relevant in cases where you expect to run many thousands of compressors concurrently, +but with very little activity. This is *not* intended for regular web servers serving individual requests. + +Because of this, the size of actual Write calls will affect output size. + +In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. + +For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) + +A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: + +```go + // replace 'ioutil.Discard' with your output. + gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) + if err != nil { + return err + } + defer gzw.Close() + + w := bufio.NewWriterSize(gzw, 4096) + defer w.Flush() + + // Write to 'w' +``` + +This will only use up to 4KB in memory when the writer is idle. + +Compression is almost always worse than the fastest compression level +and each write will allocate (a little) memory. + + +# Other packages + +Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): + +* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. +* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. +* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. +* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. +* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. +* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. +* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. + +# license + +This code is licensed under the same conditions as the original Go code. See LICENSE file. + + + + + diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md index ea7324da6..27d8ed56f 100644 --- a/vendor/github.com/klauspost/compress/fse/README.md +++ b/vendor/github.com/klauspost/compress/fse/README.md @@ -1,79 +1,79 @@ -# Finite State Entropy - -This package provides Finite State Entropy encoding and decoding. - -Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) -encoding provides a fast near-optimal symbol encoding/decoding -for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) - -## News - - * Feb 2018: First implementation released. Consider this beta software for now. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `(error)` | An internal error occurred. | - -As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). - -# Performance - -A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. -All compression functions are currently only running on the calling goroutine so only one core will be used per block. - -The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input -is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be -beneficial to transpose all your input values down by 64. - -With moderate block sizes around 64k speed are typically 200MB/s per core for compression and -around 300MB/s decompression speed. - -The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. - -# Plans - -At one point, more internals will be exposed to facilitate more "expert" usage of the components. - -A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +# Finite State Entropy + +This package provides Finite State Entropy encoding and decoding. + +Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) +encoding provides a fast near-optimal symbol encoding/decoding +for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) + +## News + + * Feb 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `(error)` | An internal error occurred. | + +As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). + +# Performance + +A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. +All compression functions are currently only running on the calling goroutine so only one core will be used per block. + +The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input +is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be +beneficial to transpose all your input values down by 64. + +With moderate block sizes around 64k speed are typically 200MB/s per core for compression and +around 300MB/s decompression speed. + +The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. + +# Plans + +At one point, more internals will be exposed to facilitate more "expert" usage of the components. + +A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md index 8b6e5c663..26d5101b3 100644 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -1,89 +1,89 @@ -# Huff0 entropy compression - -This package provides Huff0 encoding and decoding as used in zstd. - -[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), -a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU -(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) - -## News - -This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. - -This ensures that most functionality is well tested. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and -[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | -| `(error)` | An internal error occurred. | - - -As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. - -## Tables and re-use - -Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. - -The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) -that controls this behaviour. See the documentation for details. This can be altered between each block. - -Do however note that this information is *not* stored in the output block and it is up to the users of the package to -record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, -based on the boolean reported back from the CompressXX call. - -If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the -[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. - -## Decompressing - -The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). -This will initialize the decoding tables. -You can supply the complete block to `ReadTable` and it will return the data part of the block -which can be given to the decompressor. - -Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) -or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. - -For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. - -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking -changes will likely not be accepted. If in doubt open an issue before writing the PR. +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +## News + +This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. + +This ensures that most functionality is well tested. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 99ddd4af9..2d6ef64be 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // This file contains the specialisation of Decoder.Decompress4X // and Decoder.Decompress1X that use an asm implementation of thir main loops. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 908c17de6..610392322 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go index e802579c4..b97f9056f 100644 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package cpuinfo diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index fd35ea148..0e33aea44 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -78,6 +78,7 @@ func (b *blockEnc) initNewEncode() { b.recentOffsets = [3]uint32{1, 4, 8} b.litEnc.Reuse = huff0.ReusePolicyNone b.coders.setPrev(nil, nil, nil) + b.dictLitEnc = nil } // reset will reset the block for a new encode, but in the same stream, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index c1192ec38..c4de134a7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -21,7 +21,7 @@ type fastBase struct { crc *xxhash.Digest tmp [8]byte blk *blockEnc - lastDictID uint32 + lastDict *dict lowMem bool } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index c1581cfcb..851799322 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -479,10 +479,13 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]prevEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -510,13 +513,14 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -538,8 +542,8 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id } + e.lastDict = d // Reset table to initial state copy(e.longTable[:], e.dictLongTable) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 85dcd28c3..3305f0924 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -1102,10 +1102,13 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -1133,14 +1136,15 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id e.allDirty = true } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1162,9 +1166,9 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id e.allDirty = true } + e.lastDict = d // Reset table to initial state { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index cf8cad00d..2fb6da112 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -1040,15 +1040,18 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { allDirty := e.allDirty + dictChanged := d != e.lastDict e.fastEncoderDict.Reset(d, singleBlock) if d == nil { return } // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]tableEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1065,7 +1068,6 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id allDirty = true } // Reset table to initial state diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 9180a3a58..5e104f1a4 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -805,9 +805,11 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || d != e.lastDict { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } if true { end := e.maxMatchOff + int32(len(d.content)) - 8 @@ -827,7 +829,7 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id + e.lastDict = d e.allDirty = true } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 19e730acc..0f2a00a00 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -138,11 +138,18 @@ func (e *Encoder) Reset(w io.Writer) { func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error { e.o.resetOpt = true defer func() { e.o.resetOpt = false }() + hadDict := e.o.dict != nil for _, o := range opts { if err := o(&e.o); err != nil { return err } } + hasDict := e.o.dict != nil + if hadDict != hasDict { + // Dict presence changed — encoder type must be recreated. + e.state.encoder = nil + e.init = sync.Once{} + } e.Reset(w) return nil } @@ -448,6 +455,12 @@ func (e *Encoder) Close() error { if s.encoder == nil { return nil } + if s.w == nil { + if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { + return nil + } + return errors.New("zstd: encoder has no writer") + } err := e.nextBlock(true) if err != nil { if errors.Is(s.err, ErrEncoderClosed) { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 8e0f5cac7..e217be0a1 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -42,6 +42,7 @@ func (o *encoderOptions) setDefault() { level: SpeedDefault, allLitEntropy: false, lowMem: false, + fullZero: true, } } diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go index d04a829b0..b8c8607b5 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 8adfebb02..2138f8091 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 0be16cefc..9576426e6 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -1,5 +1,4 @@ //go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm -// +build !amd64,!arm64 appengine !gc purego noasm package xxhash diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go index f41932b7a..1ed18927f 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go index bea1779e9..379746c96 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 1f8c3cec2..18c3703dd 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go index 7cec2197c..516cd9b07 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd diff --git a/vendor/github.com/mitchellh/go-homedir/LICENSE b/vendor/github.com/mitchellh/go-homedir/LICENSE deleted file mode 100644 index f9c841a51..000000000 --- a/vendor/github.com/mitchellh/go-homedir/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/go-homedir/README.md b/vendor/github.com/mitchellh/go-homedir/README.md deleted file mode 100644 index d70706d5b..000000000 --- a/vendor/github.com/mitchellh/go-homedir/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# go-homedir - -This is a Go library for detecting the user's home directory without -the use of cgo, so the library can be used in cross-compilation environments. - -Usage is incredibly simple, just call `homedir.Dir()` to get the home directory -for a user, and `homedir.Expand()` to expand the `~` in a path to the home -directory. - -**Why not just use `os/user`?** The built-in `os/user` package requires -cgo on Darwin systems. This means that any Go code that uses that package -cannot cross compile. But 99% of the time the use for `os/user` is just to -retrieve the home directory, which we can do for the current user without -cgo. This library does that, enabling cross-compilation. diff --git a/vendor/github.com/mitchellh/go-homedir/homedir.go b/vendor/github.com/mitchellh/go-homedir/homedir.go deleted file mode 100644 index 25378537e..000000000 --- a/vendor/github.com/mitchellh/go-homedir/homedir.go +++ /dev/null @@ -1,167 +0,0 @@ -package homedir - -import ( - "bytes" - "errors" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" -) - -// DisableCache will disable caching of the home directory. Caching is enabled -// by default. -var DisableCache bool - -var homedirCache string -var cacheLock sync.RWMutex - -// Dir returns the home directory for the executing user. -// -// This uses an OS-specific method for discovering the home directory. -// An error is returned if a home directory cannot be detected. -func Dir() (string, error) { - if !DisableCache { - cacheLock.RLock() - cached := homedirCache - cacheLock.RUnlock() - if cached != "" { - return cached, nil - } - } - - cacheLock.Lock() - defer cacheLock.Unlock() - - var result string - var err error - if runtime.GOOS == "windows" { - result, err = dirWindows() - } else { - // Unix-like system, so just assume Unix - result, err = dirUnix() - } - - if err != nil { - return "", err - } - homedirCache = result - return result, nil -} - -// Expand expands the path to include the home directory if the path -// is prefixed with `~`. If it isn't prefixed with `~`, the path is -// returned as-is. -func Expand(path string) (string, error) { - if len(path) == 0 { - return path, nil - } - - if path[0] != '~' { - return path, nil - } - - if len(path) > 1 && path[1] != '/' && path[1] != '\\' { - return "", errors.New("cannot expand user-specific home dir") - } - - dir, err := Dir() - if err != nil { - return "", err - } - - return filepath.Join(dir, path[1:]), nil -} - -// Reset clears the cache, forcing the next call to Dir to re-detect -// the home directory. This generally never has to be called, but can be -// useful in tests if you're modifying the home directory via the HOME -// env var or something. -func Reset() { - cacheLock.Lock() - defer cacheLock.Unlock() - homedirCache = "" -} - -func dirUnix() (string, error) { - homeEnv := "HOME" - if runtime.GOOS == "plan9" { - // On plan9, env vars are lowercase. - homeEnv = "home" - } - - // First prefer the HOME environmental variable - if home := os.Getenv(homeEnv); home != "" { - return home, nil - } - - var stdout bytes.Buffer - - // If that fails, try OS specific commands - if runtime.GOOS == "darwin" { - cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`) - cmd.Stdout = &stdout - if err := cmd.Run(); err == nil { - result := strings.TrimSpace(stdout.String()) - if result != "" { - return result, nil - } - } - } else { - cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - // If the error is ErrNotFound, we ignore it. Otherwise, return it. - if err != exec.ErrNotFound { - return "", err - } - } else { - if passwd := strings.TrimSpace(stdout.String()); passwd != "" { - // username:password:uid:gid:gecos:home:shell - passwdParts := strings.SplitN(passwd, ":", 7) - if len(passwdParts) > 5 { - return passwdParts[5], nil - } - } - } - } - - // If all else fails, try the shell - stdout.Reset() - cmd := exec.Command("sh", "-c", "cd && pwd") - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - return "", err - } - - result := strings.TrimSpace(stdout.String()) - if result == "" { - return "", errors.New("blank output when reading home directory") - } - - return result, nil -} - -func dirWindows() (string, error) { - // First prefer the HOME environmental variable - if home := os.Getenv("HOME"); home != "" { - return home, nil - } - - // Prefer standard environment variable USERPROFILE - if home := os.Getenv("USERPROFILE"); home != "" { - return home, nil - } - - drive := os.Getenv("HOMEDRIVE") - path := os.Getenv("HOMEPATH") - home := drive + path - if drive == "" || path == "" { - return "", errors.New("HOMEDRIVE, HOMEPATH, or USERPROFILE are blank") - } - - return home, nil -} diff --git a/vendor/github.com/sirupsen/logrus/.golangci.yml b/vendor/github.com/sirupsen/logrus/.golangci.yml index 65dc28503..792db3618 100644 --- a/vendor/github.com/sirupsen/logrus/.golangci.yml +++ b/vendor/github.com/sirupsen/logrus/.golangci.yml @@ -1,40 +1,67 @@ +version: "2" run: - # do not run on test files yet tests: false - -# all available settings of specific linters -linters-settings: - errcheck: - # report about not checking of errors in type assetions: `a := b.(MyStruct)`; - # default is false: such cases aren't reported by default. - check-type-assertions: false - - # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; - # default is false: such cases aren't reported by default. - check-blank: false - - lll: - line-length: 100 - tab-width: 4 - - prealloc: - simple: false - range-loops: false - for-loops: false - - whitespace: - multi-if: false # Enforces newlines (or comments) after every multi-line if statement - multi-func: false # Enforces newlines (or comments) after every multi-line function signature - linters: enable: - - megacheck - - govet + - asasalint + - asciicheck + - bidichk + - bodyclose + - contextcheck + - durationcheck + - errchkjson + - errorlint + - exhaustive + - gocheckcompilerdirectives + - gochecksumtype + - gosec + - gosmopolitan + - loggercheck + - makezero + - musttag + - nilerr + - nilnesserr + - noctx + - protogetter + - reassign + - recvcheck + - rowserrcheck + - spancheck + - sqlclosecheck + - testifylint + - unparam + - zerologlint disable: - - maligned - prealloc - disable-all: false - presets: - - bugs - - unused - fast: false + settings: + errcheck: + check-type-assertions: false + check-blank: false + lll: + line-length: 100 + tab-width: 4 + prealloc: + simple: false + range-loops: false + for-loops: false + whitespace: + multi-if: false + multi-func: false + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/vendor/github.com/sirupsen/logrus/CHANGELOG.md b/vendor/github.com/sirupsen/logrus/CHANGELOG.md index 7567f6128..098608ff4 100644 --- a/vendor/github.com/sirupsen/logrus/CHANGELOG.md +++ b/vendor/github.com/sirupsen/logrus/CHANGELOG.md @@ -37,7 +37,7 @@ Features: # 1.6.0 Fixes: * end of line cleanup - * revert the entry concurrency bug fix whic leads to deadlock under some circumstances + * revert the entry concurrency bug fix which leads to deadlock under some circumstances * update dependency on go-windows-terminal-sequences to fix a crash with go 1.14 Features: @@ -129,7 +129,7 @@ This new release introduces: which is mostly useful for logger wrapper * a fix reverting the immutability of the entry given as parameter to the hooks a new configuration field of the json formatter in order to put all the fields - in a nested dictionnary + in a nested dictionary * a new SetOutput method in the Logger * a new configuration of the textformatter to configure the name of the default keys * a new configuration of the text formatter to disable the level truncation diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index d1d4a85fd..cc5dab7eb 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -1,4 +1,4 @@ -# Logrus :walrus: [![Build Status](https://github.com/sirupsen/logrus/workflows/CI/badge.svg)](https://github.com/sirupsen/logrus/actions?query=workflow%3ACI) [![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus) [![Go Reference](https://pkg.go.dev/badge/github.com/sirupsen/logrus.svg)](https://pkg.go.dev/github.com/sirupsen/logrus) +# Logrus :walrus: [![Build Status](https://github.com/sirupsen/logrus/workflows/CI/badge.svg)](https://github.com/sirupsen/logrus/actions?query=workflow%3ACI) [![Go Reference](https://pkg.go.dev/badge/github.com/sirupsen/logrus.svg)](https://pkg.go.dev/github.com/sirupsen/logrus) Logrus is a structured logger for Go (golang), completely API compatible with the standard library logger. @@ -40,7 +40,7 @@ plain text): ![Colored](http://i.imgur.com/PY7qMwd.png) -With `log.SetFormatter(&log.JSONFormatter{})`, for easy parsing by logstash +With `logrus.SetFormatter(&logrus.JSONFormatter{})`, for easy parsing by logstash or Splunk: ```text @@ -60,9 +60,9 @@ ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"} "time":"2014-03-10 19:57:38.562543128 -0400 EDT"} ``` -With the default `log.SetFormatter(&log.TextFormatter{})` when a TTY is not +With the default `logrus.SetFormatter(&logrus.TextFormatter{})` when a TTY is not attached, the output is compatible with the -[logfmt](http://godoc.org/github.com/kr/logfmt) format: +[logfmt](https://pkg.go.dev/github.com/kr/logfmt) format: ```text time="2015-03-26T01:27:38-04:00" level=debug msg="Started observing beach" animal=walrus number=8 @@ -75,17 +75,18 @@ time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x20822 To ensure this behaviour even if a TTY is attached, set your formatter as follows: ```go - log.SetFormatter(&log.TextFormatter{ - DisableColors: true, - FullTimestamp: true, - }) +logrus.SetFormatter(&logrus.TextFormatter{ + DisableColors: true, + FullTimestamp: true, +}) ``` #### Logging Method Name If you wish to add the calling method as a field, instruct the logger via: + ```go -log.SetReportCaller(true) +logrus.SetReportCaller(true) ``` This adds the caller as 'method' like so: @@ -100,11 +101,11 @@ time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcr Note that this does add measurable overhead - the cost will depend on the version of Go, but is between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your environment via benchmarks: -``` + +```bash go test -bench=.*CallerTracing ``` - #### Case-sensitivity The organization's name was changed to lower-case--and this will not be changed @@ -118,12 +119,10 @@ The simplest way to use Logrus is simply the package-level exported logger: ```go package main -import ( - log "github.com/sirupsen/logrus" -) +import "github.com/sirupsen/logrus" func main() { - log.WithFields(log.Fields{ + logrus.WithFields(logrus.Fields{ "animal": "walrus", }).Info("A walrus appears") } @@ -139,6 +138,7 @@ package main import ( "os" + log "github.com/sirupsen/logrus" ) @@ -190,26 +190,27 @@ package main import ( "os" + "github.com/sirupsen/logrus" ) // Create a new instance of the logger. You can have any number of instances. -var log = logrus.New() +var logger = logrus.New() func main() { // The API for setting attributes is a little different than the package level - // exported logger. See Godoc. - log.Out = os.Stdout + // exported logger. See Godoc. + logger.Out = os.Stdout // You could set this to any `io.Writer` such as a file // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) // if err == nil { - // log.Out = file + // logger.Out = file // } else { - // log.Info("Failed to log to file, using default stderr") + // logger.Info("Failed to log to file, using default stderr") // } - log.WithFields(logrus.Fields{ + logger.WithFields(logrus.Fields{ "animal": "walrus", "size": 10, }).Info("A group of walrus emerges from the ocean") @@ -219,12 +220,12 @@ func main() { #### Fields Logrus encourages careful, structured logging through logging fields instead of -long, unparseable error messages. For example, instead of: `log.Fatalf("Failed +long, unparseable error messages. For example, instead of: `logrus.Fatalf("Failed to send event %s to topic %s with key %d")`, you should log the much more discoverable: ```go -log.WithFields(log.Fields{ +logrus.WithFields(logrus.Fields{ "event": event, "topic": topic, "key": key, @@ -245,12 +246,12 @@ seen as a hint you should add a field, however, you can still use the Often it's helpful to have fields _always_ attached to log statements in an application or parts of one. For example, you may want to always log the `request_id` and `user_ip` in the context of a request. Instead of writing -`log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})` on +`logger.WithFields(logrus.Fields{"request_id": request_id, "user_ip": user_ip})` on every line, you can create a `logrus.Entry` to pass around instead: ```go -requestLogger := log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip}) -requestLogger.Info("something happened on that request") # will log request_id and user_ip +requestLogger := logger.WithFields(logrus.Fields{"request_id": request_id, "user_ip": user_ip}) +requestLogger.Info("something happened on that request") // will log request_id and user_ip requestLogger.Warn("something not great happened") ``` @@ -264,28 +265,31 @@ Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in `init`: ```go +package main + import ( - log "github.com/sirupsen/logrus" - "gopkg.in/gemnasium/logrus-airbrake-hook.v2" // the package is named "airbrake" - logrus_syslog "github.com/sirupsen/logrus/hooks/syslog" "log/syslog" + + "github.com/sirupsen/logrus" + airbrake "gopkg.in/gemnasium/logrus-airbrake-hook.v2" + logrus_syslog "github.com/sirupsen/logrus/hooks/syslog" ) func init() { // Use the Airbrake hook to report errors that have Error severity or above to // an exception tracker. You can create custom hooks, see the Hooks section. - log.AddHook(airbrake.NewHook(123, "xyz", "production")) + logrus.AddHook(airbrake.NewHook(123, "xyz", "production")) hook, err := logrus_syslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, "") if err != nil { - log.Error("Unable to connect to local syslog daemon") + logrus.Error("Unable to connect to local syslog daemon") } else { - log.AddHook(hook) + logrus.AddHook(hook) } } ``` -Note: Syslog hook also support connecting to local syslog (Ex. "/dev/log" or "/var/run/syslog" or "/var/run/log"). For the detail, please check the [syslog hook README](hooks/syslog/README.md). +Note: Syslog hooks also support connecting to local syslog (Ex. "/dev/log" or "/var/run/syslog" or "/var/run/log"). For the detail, please check the [syslog hook README](hooks/syslog/README.md). A list of currently known service hooks can be found in this wiki [page](https://github.com/sirupsen/logrus/wiki/Hooks) @@ -295,15 +299,15 @@ A list of currently known service hooks can be found in this wiki [page](https:/ Logrus has seven logging levels: Trace, Debug, Info, Warning, Error, Fatal and Panic. ```go -log.Trace("Something very low level.") -log.Debug("Useful debugging information.") -log.Info("Something noteworthy happened!") -log.Warn("You should probably take a look at this.") -log.Error("Something failed but I'm not quitting.") +logrus.Trace("Something very low level.") +logrus.Debug("Useful debugging information.") +logrus.Info("Something noteworthy happened!") +logrus.Warn("You should probably take a look at this.") +logrus.Error("Something failed but I'm not quitting.") // Calls os.Exit(1) after logging -log.Fatal("Bye.") +logrus.Fatal("Bye.") // Calls panic() after logging -log.Panic("I'm bailing.") +logrus.Panic("I'm bailing.") ``` You can set the logging level on a `Logger`, then it will only log entries with @@ -311,13 +315,13 @@ that severity or anything above it: ```go // Will log anything that is info or above (warn, error, fatal, panic). Default. -log.SetLevel(log.InfoLevel) +logrus.SetLevel(logrus.InfoLevel) ``` -It may be useful to set `log.Level = logrus.DebugLevel` in a debug or verbose +It may be useful to set `logrus.Level = logrus.DebugLevel` in a debug or verbose environment if your application has that. -Note: If you want different log levels for global (`log.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging). +Note: If you want different log levels for global (`logrus.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging). #### Entries @@ -340,17 +344,17 @@ could do: ```go import ( - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" ) func init() { // do something here to set environment depending on an environment variable // or command-line flag if Environment == "production" { - log.SetFormatter(&log.JSONFormatter{}) + logrus.SetFormatter(&logrus.JSONFormatter{}) } else { // The TextFormatter is default, you don't actually have to do this. - log.SetFormatter(&log.TextFormatter{}) + logrus.SetFormatter(&logrus.TextFormatter{}) } } ``` @@ -372,11 +376,11 @@ The built-in logging formatters are: * When colors are enabled, levels are truncated to 4 characters by default. To disable truncation set the `DisableLevelTruncation` field to `true`. * When outputting to a TTY, it's often helpful to visually scan down a column where all the levels are the same width. Setting the `PadLevelText` field to `true` enables this behavior, by adding padding to the level text. - * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#TextFormatter). + * All options are listed in the [generated docs](https://pkg.go.dev/github.com/sirupsen/logrus#TextFormatter). * `logrus.JSONFormatter`. Logs fields as JSON. - * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#JSONFormatter). + * All options are listed in the [generated docs](https://pkg.go.dev/github.com/sirupsen/logrus#JSONFormatter). -Third party logging formatters: +Third-party logging formatters: * [`FluentdFormatter`](https://github.com/joonix/log). Formats entries that can be parsed by Kubernetes and Google Container Engine. * [`GELF`](https://github.com/fabienm/go-logrus-formatters). Formats entries so they comply to Graylog's [GELF 1.1 specification](http://docs.graylog.org/en/2.4/pages/gelf.html). @@ -384,7 +388,7 @@ Third party logging formatters: * [`prefixed`](https://github.com/x-cray/logrus-prefixed-formatter). Displays log entry source along with alternative layout. * [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the Power of Zalgo. * [`nested-logrus-formatter`](https://github.com/antonfisher/nested-logrus-formatter). Converts logrus fields to a nested structure. -* [`powerful-logrus-formatter`](https://github.com/zput/zxcTool). get fileName, log's line number and the latest function's name when print log; Sava log to files. +* [`powerful-logrus-formatter`](https://github.com/zput/zxcTool). get fileName, log's line number and the latest function's name when print log; Save log to files. * [`caption-json-formatter`](https://github.com/nolleh/caption_json_formatter). logrus's message json formatter with human-readable caption added. You can define your formatter by implementing the `Formatter` interface, @@ -393,10 +397,9 @@ requiring a `Format` method. `Format` takes an `*Entry`. `entry.Data` is a default ones (see Entries section above): ```go -type MyJSONFormatter struct { -} +type MyJSONFormatter struct{} -log.SetFormatter(new(MyJSONFormatter)) +logrus.SetFormatter(new(MyJSONFormatter)) func (f *MyJSONFormatter) Format(entry *Entry) ([]byte, error) { // Note this doesn't include Time, Level and Message which are available on @@ -455,17 +458,18 @@ entries. It should not be a feature of the application-level logger. #### Testing -Logrus has a built in facility for asserting the presence of log messages. This is implemented through the `test` hook and provides: +Logrus has a built-in facility for asserting the presence of log messages. This is implemented through the `test` hook and provides: * decorators for existing logger (`test.NewLocal` and `test.NewGlobal`) which basically just adds the `test` hook * a test logger (`test.NewNullLogger`) that just records log messages (and does not output any): ```go import( + "testing" + "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" - "testing" ) func TestSomething(t*testing.T){ @@ -486,15 +490,15 @@ func TestSomething(t*testing.T){ Logrus can register one or more functions that will be called when any `fatal` level message is logged. The registered handlers will be executed before logrus performs an `os.Exit(1)`. This behavior may be helpful if callers need -to gracefully shutdown. Unlike a `panic("Something went wrong...")` call which can be intercepted with a deferred `recover` a call to `os.Exit(1)` can not be intercepted. +to gracefully shut down. Unlike a `panic("Something went wrong...")` call which can be intercepted with a deferred `recover` a call to `os.Exit(1)` can not be intercepted. -``` -... +```go +// ... handler := func() { - // gracefully shutdown something... + // gracefully shut down something... } logrus.RegisterExitHandler(handler) -... +// ... ``` #### Thread safety @@ -502,7 +506,7 @@ logrus.RegisterExitHandler(handler) By default, Logger is protected by a mutex for concurrent writes. The mutex is held when calling hooks and writing logs. If you are sure such locking is not needed, you can call logger.SetNoLock() to disable the locking. -Situation when locking is not needed includes: +Situations when locking is not needed include: * You have no hooks registered, or hooks calling is already thread-safe. diff --git a/vendor/github.com/sirupsen/logrus/appveyor.yml b/vendor/github.com/sirupsen/logrus/appveyor.yml index df9d65c3a..e90f09ea6 100644 --- a/vendor/github.com/sirupsen/logrus/appveyor.yml +++ b/vendor/github.com/sirupsen/logrus/appveyor.yml @@ -1,14 +1,12 @@ -version: "{build}" +# Minimal stub to satisfy AppVeyor CI +version: 1.0.{build} platform: x64 -clone_folder: c:\gopath\src\github.com\sirupsen\logrus -environment: - GOPATH: c:\gopath +shallow_clone: true + branches: only: - master -install: - - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - - go version + - main + build_script: - - go get -t - - go test + - echo "No-op build to satisfy AppVeyor CI" diff --git a/vendor/github.com/sirupsen/logrus/entry.go b/vendor/github.com/sirupsen/logrus/entry.go index 71cdbbc35..71d796d0b 100644 --- a/vendor/github.com/sirupsen/logrus/entry.go +++ b/vendor/github.com/sirupsen/logrus/entry.go @@ -34,13 +34,15 @@ func init() { minimumCallerDepth = 1 } -// Defines the key when adding errors using WithError. +// ErrorKey defines the key when adding errors using [WithError], [Logger.WithError]. var ErrorKey = "error" -// An entry is the final or intermediate Logrus logging entry. It contains all +// Entry is the final or intermediate Logrus logging entry. It contains all // the fields passed with WithField{,s}. It's finally logged when Trace, Debug, // Info, Warn, Error, Fatal or Panic is called on it. These objects can be // reused and passed around as much as you wish to avoid field duplication. +// +//nolint:recvcheck // the methods of "Entry" use pointer receiver and non-pointer receiver. type Entry struct { Logger *Logger @@ -86,12 +88,12 @@ func (entry *Entry) Dup() *Entry { return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time, Context: entry.Context, err: entry.err} } -// Returns the bytes representation of this entry from the formatter. +// Bytes returns the bytes representation of this entry from the formatter. func (entry *Entry) Bytes() ([]byte, error) { return entry.Logger.Formatter.Format(entry) } -// Returns the string representation from the reader and ultimately the +// String returns the string representation from the reader and ultimately the // formatter. func (entry *Entry) String() (string, error) { serialized, err := entry.Bytes() @@ -102,12 +104,13 @@ func (entry *Entry) String() (string, error) { return str, nil } -// Add an error as single field (using the key defined in ErrorKey) to the Entry. +// WithError adds an error as single field (using the key defined in [ErrorKey]) +// to the Entry. func (entry *Entry) WithError(err error) *Entry { return entry.WithField(ErrorKey, err) } -// Add a context to the Entry. +// WithContext adds a context to the Entry. func (entry *Entry) WithContext(ctx context.Context) *Entry { dataCopy := make(Fields, len(entry.Data)) for k, v := range entry.Data { @@ -116,12 +119,12 @@ func (entry *Entry) WithContext(ctx context.Context) *Entry { return &Entry{Logger: entry.Logger, Data: dataCopy, Time: entry.Time, err: entry.err, Context: ctx} } -// Add a single field to the Entry. +// WithField adds a single field to the Entry. func (entry *Entry) WithField(key string, value interface{}) *Entry { return entry.WithFields(Fields{key: value}) } -// Add a map of fields to the Entry. +// WithFields adds a map of fields to the Entry. func (entry *Entry) WithFields(fields Fields) *Entry { data := make(Fields, len(entry.Data)+len(fields)) for k, v := range entry.Data { @@ -150,7 +153,7 @@ func (entry *Entry) WithFields(fields Fields) *Entry { return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time, err: fieldErr, Context: entry.Context} } -// Overrides the time of the Entry. +// WithTime overrides the time of the Entry. func (entry *Entry) WithTime(t time.Time) *Entry { dataCopy := make(Fields, len(entry.Data)) for k, v := range entry.Data { @@ -204,7 +207,7 @@ func getCaller() *runtime.Frame { // If the caller isn't part of this package, we're done if pkg != logrusPackage { - return &f //nolint:scopelint + return &f } } @@ -432,7 +435,7 @@ func (entry *Entry) Panicln(args ...interface{}) { entry.Logln(PanicLevel, args...) } -// Sprintlnn => Sprint no newline. This is to get the behavior of how +// sprintlnn => Sprint no newline. This is to get the behavior of how // fmt.Sprintln where spaces are always added between operands, regardless of // their type. Instead of vendoring the Sprintln implementation to spare a // string allocation, we do the simplest thing. diff --git a/vendor/github.com/sirupsen/logrus/hooks.go b/vendor/github.com/sirupsen/logrus/hooks.go index 3f151cdc3..9ab978a45 100644 --- a/vendor/github.com/sirupsen/logrus/hooks.go +++ b/vendor/github.com/sirupsen/logrus/hooks.go @@ -1,16 +1,16 @@ package logrus -// A hook to be fired when logging on the logging levels returned from -// `Levels()` on your implementation of the interface. Note that this is not +// Hook describes hooks to be fired when logging on the logging levels returned from +// [Hook.Levels] on your implementation of the interface. Note that this is not // fired in a goroutine or a channel with workers, you should handle such -// functionality yourself if your call is non-blocking and you don't wish for +// functionality yourself if your call is non-blocking, and you don't wish for // the logging calls for levels returned from `Levels()` to block. type Hook interface { Levels() []Level Fire(*Entry) error } -// Internal type for storing the hooks on a logger instance. +// LevelHooks is an internal type for storing the hooks on a logger instance. type LevelHooks map[Level][]Hook // Add a hook to an instance of logger. This is called with diff --git a/vendor/github.com/sirupsen/logrus/logger.go b/vendor/github.com/sirupsen/logrus/logger.go index 5ff0aef6d..f5b8c439e 100644 --- a/vendor/github.com/sirupsen/logrus/logger.go +++ b/vendor/github.com/sirupsen/logrus/logger.go @@ -72,16 +72,16 @@ func (mw *MutexWrap) Disable() { mw.disabled = true } -// Creates a new logger. Configuration should be set by changing `Formatter`, -// `Out` and `Hooks` directly on the default logger instance. You can also just +// New Creates a new logger. Configuration should be set by changing [Formatter], +// Out and Hooks directly on the default Logger instance. You can also just // instantiate your own: // -// var log = &logrus.Logger{ -// Out: os.Stderr, -// Formatter: new(logrus.TextFormatter), -// Hooks: make(logrus.LevelHooks), -// Level: logrus.DebugLevel, -// } +// var log = &logrus.Logger{ +// Out: os.Stderr, +// Formatter: new(logrus.TextFormatter), +// Hooks: make(logrus.LevelHooks), +// Level: logrus.DebugLevel, +// } // // It's recommended to make this a global instance called `log`. func New() *Logger { @@ -118,30 +118,30 @@ func (logger *Logger) WithField(key string, value interface{}) *Entry { return entry.WithField(key, value) } -// Adds a struct of fields to the log entry. All it does is call `WithField` for -// each `Field`. +// WithFields adds a struct of fields to the log entry. It calls [Entry.WithField] +// for each Field. func (logger *Logger) WithFields(fields Fields) *Entry { entry := logger.newEntry() defer logger.releaseEntry(entry) return entry.WithFields(fields) } -// Add an error as single field to the log entry. All it does is call -// `WithError` for the given `error`. +// WithError adds an error as single field to the log entry. It calls +// [Entry.WithError] for the given error. func (logger *Logger) WithError(err error) *Entry { entry := logger.newEntry() defer logger.releaseEntry(entry) return entry.WithError(err) } -// Add a context to the log entry. +// WithContext add a context to the log entry. func (logger *Logger) WithContext(ctx context.Context) *Entry { entry := logger.newEntry() defer logger.releaseEntry(entry) return entry.WithContext(ctx) } -// Overrides the time of the log entry. +// WithTime overrides the time of the log entry. func (logger *Logger) WithTime(t time.Time) *Entry { entry := logger.newEntry() defer logger.releaseEntry(entry) @@ -347,9 +347,9 @@ func (logger *Logger) Exit(code int) { logger.ExitFunc(code) } -//When file is opened with appending mode, it's safe to -//write concurrently to a file (within 4k message on Linux). -//In these cases user can choose to disable the lock. +// SetNoLock disables the lock for situations where a file is opened with +// appending mode, and safe for concurrent writes to the file (within 4k +// message on Linux). In these cases user can choose to disable the lock. func (logger *Logger) SetNoLock() { logger.mu.Disable() } diff --git a/vendor/github.com/sirupsen/logrus/logrus.go b/vendor/github.com/sirupsen/logrus/logrus.go index 2f16224cb..37fc4fef8 100644 --- a/vendor/github.com/sirupsen/logrus/logrus.go +++ b/vendor/github.com/sirupsen/logrus/logrus.go @@ -6,13 +6,15 @@ import ( "strings" ) -// Fields type, used to pass to `WithFields`. +// Fields type, used to pass to [WithFields]. type Fields map[string]interface{} // Level type +// +//nolint:recvcheck // the methods of "Entry" use pointer receiver and non-pointer receiver. type Level uint32 -// Convert the Level to a string. E.g. PanicLevel becomes "panic". +// Convert the Level to a string. E.g. [PanicLevel] becomes "panic". func (level Level) String() string { if b, err := level.MarshalText(); err == nil { return string(b) @@ -77,7 +79,7 @@ func (level Level) MarshalText() ([]byte, error) { return nil, fmt.Errorf("not a valid logrus level %d", level) } -// A constant exposing all logging levels +// AllLevels exposing all logging levels. var AllLevels = []Level{ PanicLevel, FatalLevel, @@ -119,8 +121,8 @@ var ( ) // StdLogger is what your logrus-enabled library should take, that way -// it'll accept a stdlib logger and a logrus logger. There's no standard -// interface, this is the closest we get, unfortunately. +// it'll accept a stdlib logger ([log.Logger]) and a logrus logger. +// There's no standard interface, so this is the closest we get, unfortunately. type StdLogger interface { Print(...interface{}) Printf(string, ...interface{}) @@ -135,7 +137,8 @@ type StdLogger interface { Panicln(...interface{}) } -// The FieldLogger interface generalizes the Entry and Logger types +// FieldLogger extends the [StdLogger] interface, generalizing +// the [Entry] and [Logger] types. type FieldLogger interface { WithField(key string, value interface{}) *Entry WithFields(fields Fields) *Entry @@ -176,8 +179,9 @@ type FieldLogger interface { // IsPanicEnabled() bool } -// Ext1FieldLogger (the first extension to FieldLogger) is superfluous, it is -// here for consistancy. Do not use. Use Logger or Entry instead. +// Ext1FieldLogger (the first extension to [FieldLogger]) is superfluous, it is +// here for consistency. Do not use. Use [FieldLogger], [Logger] or [Entry] +// instead. type Ext1FieldLogger interface { FieldLogger Tracef(format string, args ...interface{}) diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go b/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go index 499789984..69956b425 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go @@ -1,4 +1,4 @@ -// +build darwin dragonfly freebsd netbsd openbsd +// +build darwin dragonfly freebsd netbsd openbsd hurd // +build !js package logrus diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_unix.go b/vendor/github.com/sirupsen/logrus/terminal_check_unix.go index 04748b851..c9aed267a 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_unix.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_unix.go @@ -1,5 +1,7 @@ +//go:build (linux || aix || zos) && !js && !wasi // +build linux aix zos // +build !js +// +build !wasi package logrus diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_wasi.go b/vendor/github.com/sirupsen/logrus/terminal_check_wasi.go new file mode 100644 index 000000000..2822b212f --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_wasi.go @@ -0,0 +1,8 @@ +//go:build wasi +// +build wasi + +package logrus + +func isTerminal(fd int) bool { + return false +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_wasip1.go b/vendor/github.com/sirupsen/logrus/terminal_check_wasip1.go new file mode 100644 index 000000000..108a6be12 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_wasip1.go @@ -0,0 +1,8 @@ +//go:build wasip1 +// +build wasip1 + +package logrus + +func isTerminal(fd int) bool { + return false +} diff --git a/vendor/github.com/sirupsen/logrus/text_formatter.go b/vendor/github.com/sirupsen/logrus/text_formatter.go index be2c6efe5..6dfeb18b1 100644 --- a/vendor/github.com/sirupsen/logrus/text_formatter.go +++ b/vendor/github.com/sirupsen/logrus/text_formatter.go @@ -306,6 +306,7 @@ func (f *TextFormatter) needsQuoting(text string) bool { return false } for _, ch := range text { + //nolint:staticcheck // QF1001: could apply De Morgan's law if !((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || @@ -334,6 +335,6 @@ func (f *TextFormatter) appendValue(b *bytes.Buffer, value interface{}) { if !f.needsQuoting(stringVal) { b.WriteString(stringVal) } else { - b.WriteString(fmt.Sprintf("%q", stringVal)) + fmt.Fprintf(b, "%q", stringVal) } } diff --git a/vendor/github.com/vbatts/tar-split/LICENSE b/vendor/github.com/vbatts/tar-split/LICENSE deleted file mode 100644 index ca03685b1..000000000 --- a/vendor/github.com/vbatts/tar-split/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/common.go b/vendor/github.com/vbatts/tar-split/archive/tar/common.go deleted file mode 100644 index e687a08c9..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/common.go +++ /dev/null @@ -1,724 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package tar implements access to tar archives. -// -// Tape archives (tar) are a file format for storing a sequence of files that -// can be read and written in a streaming manner. -// This package aims to cover most variations of the format, -// including those produced by GNU and BSD tar tools. -package tar - -import ( - "errors" - "fmt" - "math" - "os" - "path" - "reflect" - "strconv" - "strings" - "time" -) - -// BUG: Use of the Uid and Gid fields in Header could overflow on 32-bit -// architectures. If a large value is encountered when decoding, the result -// stored in Header will be the truncated version. - -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errMissData = errors.New("archive/tar: sparse file references non-existent data") - errUnrefData = errors.New("archive/tar: sparse file contains unreferenced data") - errWriteHole = errors.New("archive/tar: write non-NUL byte in sparse hole") - errSparseTooLong = errors.New("archive/tar: sparse map too long") -) - -type headerError []string - -func (he headerError) Error() string { - const prefix = "archive/tar: cannot encode header" - var ss []string - for _, s := range he { - if s != "" { - ss = append(ss, s) - } - } - if len(ss) == 0 { - return prefix - } - return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and ")) -} - -// Type flags for Header.Typeflag. -const ( - // Type '0' indicates a regular file. - TypeReg = '0' - TypeRegA = '\x00' // Deprecated: Use TypeReg instead. - - // Type '1' to '6' are header-only flags and may not have a data body. - TypeLink = '1' // Hard link - TypeSymlink = '2' // Symbolic link - TypeChar = '3' // Character device node - TypeBlock = '4' // Block device node - TypeDir = '5' // Directory - TypeFifo = '6' // FIFO node - - // Type '7' is reserved. - TypeCont = '7' - - // Type 'x' is used by the PAX format to store key-value records that - // are only relevant to the next file. - // This package transparently handles these types. - TypeXHeader = 'x' - - // Type 'g' is used by the PAX format to store key-value records that - // are relevant to all subsequent files. - // This package only supports parsing and composing such headers, - // but does not currently support persisting the global state across files. - TypeXGlobalHeader = 'g' - - // Type 'S' indicates a sparse file in the GNU format. - TypeGNUSparse = 'S' - - // Types 'L' and 'K' are used by the GNU format for a meta file - // used to store the path or link name for the next file. - // This package transparently handles these types. - TypeGNULongName = 'L' - TypeGNULongLink = 'K' -) - -// Keywords for PAX extended header records. -const ( - paxNone = "" // Indicates that no PAX key is suitable - paxPath = "path" - paxLinkpath = "linkpath" - paxSize = "size" - paxUid = "uid" - paxGid = "gid" - paxUname = "uname" - paxGname = "gname" - paxMtime = "mtime" - paxAtime = "atime" - paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid - paxCharset = "charset" // Currently unused - paxComment = "comment" // Currently unused - - paxSchilyXattr = "SCHILY.xattr." - - // Keywords for GNU sparse files in a PAX extended header. - paxGNUSparse = "GNU.sparse." - paxGNUSparseNumBlocks = "GNU.sparse.numblocks" - paxGNUSparseOffset = "GNU.sparse.offset" - paxGNUSparseNumBytes = "GNU.sparse.numbytes" - paxGNUSparseMap = "GNU.sparse.map" - paxGNUSparseName = "GNU.sparse.name" - paxGNUSparseMajor = "GNU.sparse.major" - paxGNUSparseMinor = "GNU.sparse.minor" - paxGNUSparseSize = "GNU.sparse.size" - paxGNUSparseRealSize = "GNU.sparse.realsize" -) - -// basicKeys is a set of the PAX keys for which we have built-in support. -// This does not contain "charset" or "comment", which are both PAX-specific, -// so adding them as first-class features of Header is unlikely. -// Users can use the PAXRecords field to set it themselves. -var basicKeys = map[string]bool{ - paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true, - paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true, -} - -// A Header represents a single header in a tar archive. -// Some fields may not be populated. -// -// For forward compatibility, users that retrieve a Header from Reader.Next, -// mutate it in some ways, and then pass it back to Writer.WriteHeader -// should do so by creating a new Header and copying the fields -// that they are interested in preserving. -type Header struct { - // Typeflag is the type of header entry. - // The zero value is automatically promoted to either TypeReg or TypeDir - // depending on the presence of a trailing slash in Name. - Typeflag byte - - Name string // Name of file entry - Linkname string // Target name of link (valid for TypeLink or TypeSymlink) - - Size int64 // Logical file size in bytes - Mode int64 // Permission and mode bits - Uid int // User ID of owner - Gid int // Group ID of owner - Uname string // User name of owner - Gname string // Group name of owner - - // If the Format is unspecified, then Writer.WriteHeader rounds ModTime - // to the nearest second and ignores the AccessTime and ChangeTime fields. - // - // To use AccessTime or ChangeTime, specify the Format as PAX or GNU. - // To use sub-second resolution, specify the Format as PAX. - ModTime time.Time // Modification time - AccessTime time.Time // Access time (requires either PAX or GNU support) - ChangeTime time.Time // Change time (requires either PAX or GNU support) - - Devmajor int64 // Major device number (valid for TypeChar or TypeBlock) - Devminor int64 // Minor device number (valid for TypeChar or TypeBlock) - - // Xattrs stores extended attributes as PAX records under the - // "SCHILY.xattr." namespace. - // - // The following are semantically equivalent: - // h.Xattrs[key] = value - // h.PAXRecords["SCHILY.xattr."+key] = value - // - // When Writer.WriteHeader is called, the contents of Xattrs will take - // precedence over those in PAXRecords. - // - // Deprecated: Use PAXRecords instead. - Xattrs map[string]string - - // PAXRecords is a map of PAX extended header records. - // - // User-defined records should have keys of the following form: - // VENDOR.keyword - // Where VENDOR is some namespace in all uppercase, and keyword may - // not contain the '=' character (e.g., "GOLANG.pkg.version"). - // The key and value should be non-empty UTF-8 strings. - // - // When Writer.WriteHeader is called, PAX records derived from the - // other fields in Header take precedence over PAXRecords. - PAXRecords map[string]string - - // Format specifies the format of the tar header. - // - // This is set by Reader.Next as a best-effort guess at the format. - // Since the Reader liberally reads some non-compliant files, - // it is possible for this to be FormatUnknown. - // - // If the format is unspecified when Writer.WriteHeader is called, - // then it uses the first format (in the order of USTAR, PAX, GNU) - // capable of encoding this Header (see Format). - Format Format -} - -// sparseEntry represents a Length-sized fragment at Offset in the file. -type sparseEntry struct{ Offset, Length int64 } - -func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length } - -// A sparse file can be represented as either a sparseDatas or a sparseHoles. -// As long as the total size is known, they are equivalent and one can be -// converted to the other form and back. The various tar formats with sparse -// file support represent sparse files in the sparseDatas form. That is, they -// specify the fragments in the file that has data, and treat everything else as -// having zero bytes. As such, the encoding and decoding logic in this package -// deals with sparseDatas. -// -// However, the external API uses sparseHoles instead of sparseDatas because the -// zero value of sparseHoles logically represents a normal file (i.e., there are -// no holes in it). On the other hand, the zero value of sparseDatas implies -// that the file has no data in it, which is rather odd. -// -// As an example, if the underlying raw file contains the 10-byte data: -// var compactFile = "abcdefgh" -// -// And the sparse map has the following entries: -// var spd sparseDatas = []sparseEntry{ -// {Offset: 2, Length: 5}, // Data fragment for 2..6 -// {Offset: 18, Length: 3}, // Data fragment for 18..20 -// } -// var sph sparseHoles = []sparseEntry{ -// {Offset: 0, Length: 2}, // Hole fragment for 0..1 -// {Offset: 7, Length: 11}, // Hole fragment for 7..17 -// {Offset: 21, Length: 4}, // Hole fragment for 21..24 -// } -// -// Then the content of the resulting sparse file with a Header.Size of 25 is: -// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 -type ( - sparseDatas []sparseEntry - sparseHoles []sparseEntry -) - -// validateSparseEntries reports whether sp is a valid sparse map. -// It does not matter whether sp represents data fragments or hole fragments. -func validateSparseEntries(sp []sparseEntry, size int64) bool { - // Validate all sparse entries. These are the same checks as performed by - // the BSD tar utility. - if size < 0 { - return false - } - var pre sparseEntry - for _, cur := range sp { - switch { - case cur.Offset < 0 || cur.Length < 0: - return false // Negative values are never okay - case cur.Offset > math.MaxInt64-cur.Length: - return false // Integer overflow with large length - case cur.endOffset() > size: - return false // Region extends beyond the actual size - case pre.endOffset() > cur.Offset: - return false // Regions cannot overlap and must be in order - } - pre = cur - } - return true -} - -// alignSparseEntries mutates src and returns dst where each fragment's -// starting offset is aligned up to the nearest block edge, and each -// ending offset is aligned down to the nearest block edge. -// -// Even though the Go tar Reader and the BSD tar utility can handle entries -// with arbitrary offsets and lengths, the GNU tar utility can only handle -// offsets and lengths that are multiples of blockSize. -func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - for _, s := range src { - pos, end := s.Offset, s.endOffset() - pos += blockPadding(+pos) // Round-up to nearest blockSize - if end != size { - end -= blockPadding(-end) // Round-down to nearest blockSize - } - if pos < end { - dst = append(dst, sparseEntry{Offset: pos, Length: end - pos}) - } - } - return dst -} - -// invertSparseEntries converts a sparse map from one form to the other. -// If the input is sparseHoles, then it will output sparseDatas and vice-versa. -// The input must have been already validated. -// -// This function mutates src and returns a normalized map where: -// * adjacent fragments are coalesced together -// * only the last fragment may be empty -// * the endOffset of the last fragment is the total size -func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - var pre sparseEntry - for _, cur := range src { - if cur.Length == 0 { - continue // Skip empty fragments - } - pre.Length = cur.Offset - pre.Offset - if pre.Length > 0 { - dst = append(dst, pre) // Only add non-empty fragments - } - pre.Offset = cur.endOffset() - } - pre.Length = size - pre.Offset // Possibly the only empty fragment - return append(dst, pre) -} - -// fileState tracks the number of logical (includes sparse holes) and physical -// (actual in tar archive) bytes remaining for the current file. -// -// Invariant: LogicalRemaining >= PhysicalRemaining -type fileState interface { - LogicalRemaining() int64 - PhysicalRemaining() int64 -} - -// allowedFormats determines which formats can be used. -// The value returned is the logical OR of multiple possible formats. -// If the value is FormatUnknown, then the input Header cannot be encoded -// and an error is returned explaining why. -// -// As a by-product of checking the fields, this function returns paxHdrs, which -// contain all fields that could not be directly encoded. -// A value receiver ensures that this method does not mutate the source Header. -func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) { - format = FormatUSTAR | FormatPAX | FormatGNU - paxHdrs = make(map[string]string) - - var whyNoUSTAR, whyNoPAX, whyNoGNU string - var preferPAX bool // Prefer PAX over USTAR - verifyString := func(s string, size int, name, paxKey string) { - // NUL-terminator is optional for path and linkpath. - // Technically, it is required for uname and gname, - // but neither GNU nor BSD tar checks for it. - tooLong := len(s) > size - allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath - if hasNUL(s) || (tooLong && !allowLongGNU) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s) - format.mustNotBe(FormatGNU) - } - if !isASCII(s) || tooLong { - canSplitUSTAR := paxKey == paxPath - if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s) - format.mustNotBe(FormatUSTAR) - } - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = s - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == s { - paxHdrs[paxKey] = v - } - } - verifyNumeric := func(n int64, size int, name, paxKey string) { - if !fitsInBase256(size, n) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n) - format.mustNotBe(FormatGNU) - } - if !fitsInOctal(size, n) { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n) - format.mustNotBe(FormatUSTAR) - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = strconv.FormatInt(n, 10) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) { - paxHdrs[paxKey] = v - } - } - verifyTime := func(ts time.Time, size int, name, paxKey string) { - if ts.IsZero() { - return // Always okay - } - if !fitsInBase256(size, ts.Unix()) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts) - format.mustNotBe(FormatGNU) - } - isMtime := paxKey == paxMtime - fitsOctal := fitsInOctal(size, ts.Unix()) - if (isMtime && !fitsOctal) || !isMtime { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts) - format.mustNotBe(FormatUSTAR) - } - needsNano := ts.Nanosecond() != 0 - if !isMtime || !fitsOctal || needsNano { - preferPAX = true // USTAR may truncate sub-second measurements - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = formatPAXTime(ts) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) { - paxHdrs[paxKey] = v - } - } - - // Check basic fields. - var blk block - v7 := blk.V7() - ustar := blk.USTAR() - gnu := blk.GNU() - verifyString(h.Name, len(v7.Name()), "Name", paxPath) - verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath) - verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname) - verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname) - verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone) - verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid) - verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid) - verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize) - verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone) - verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone) - verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime) - verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime) - verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime) - - // Check for header-only types. - var whyOnlyPAX, whyOnlyGNU string - switch h.Typeflag { - case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse: - // Exclude TypeLink and TypeSymlink, since they may reference directories. - if strings.HasSuffix(h.Name, "/") { - return FormatUnknown, nil, headerError{"filename may not have trailing slash"} - } - case TypeXHeader, TypeGNULongName, TypeGNULongLink: - return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"} - case TypeXGlobalHeader: - h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format} - if !reflect.DeepEqual(h, h2) { - return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"} - } - whyOnlyPAX = "only PAX supports TypeXGlobalHeader" - format.mayOnlyBe(FormatPAX) - } - if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 { - return FormatUnknown, nil, headerError{"negative size on header-only type"} - } - - // Check PAX records. - if len(h.Xattrs) > 0 { - for k, v := range h.Xattrs { - paxHdrs[paxSchilyXattr+k] = v - } - whyOnlyPAX = "only PAX supports Xattrs" - format.mayOnlyBe(FormatPAX) - } - if len(h.PAXRecords) > 0 { - for k, v := range h.PAXRecords { - switch _, exists := paxHdrs[k]; { - case exists: - continue // Do not overwrite existing records - case h.Typeflag == TypeXGlobalHeader: - paxHdrs[k] = v // Copy all records - case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse): - paxHdrs[k] = v // Ignore local records that may conflict - } - } - whyOnlyPAX = "only PAX supports PAXRecords" - format.mayOnlyBe(FormatPAX) - } - for k, v := range paxHdrs { - if !validPAXRecord(k, v) { - return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)} - } - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Check sparse files. - if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse { - if isHeaderOnlyType(h.Typeflag) { - return FormatUnknown, nil, headerError{"header-only type cannot be sparse"} - } - if !validateSparseEntries(h.SparseHoles, h.Size) { - return FormatUnknown, nil, headerError{"invalid sparse holes"} - } - if h.Typeflag == TypeGNUSparse { - whyOnlyGNU = "only GNU supports TypeGNUSparse" - format.mayOnlyBe(FormatGNU) - } else { - whyNoGNU = "GNU supports sparse files only with TypeGNUSparse" - format.mustNotBe(FormatGNU) - } - whyNoUSTAR = "USTAR does not support sparse files" - format.mustNotBe(FormatUSTAR) - } - */ - - // Check desired format. - if wantFormat := h.Format; wantFormat != FormatUnknown { - if wantFormat.has(FormatPAX) && !preferPAX { - wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too - } - format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted - } - if format == FormatUnknown { - switch h.Format { - case FormatUSTAR: - err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU} - case FormatPAX: - err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU} - case FormatGNU: - err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX} - default: - err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU} - } - } - return format, paxHdrs, err -} - -// FileInfo returns an os.FileInfo for the Header. -func (h *Header) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - h *Header -} - -func (fi headerFileInfo) Size() int64 { return fi.h.Size } -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } -func (fi headerFileInfo) Sys() interface{} { return fi.h } - -// Name returns the base name of the file. -func (fi headerFileInfo) Name() string { - if fi.IsDir() { - return path.Base(path.Clean(fi.h.Name)) - } - return path.Base(fi.h.Name) -} - -// Mode returns the permission and mode bits for the headerFileInfo. -func (fi headerFileInfo) Mode() (mode os.FileMode) { - // Set file permission bits. - mode = os.FileMode(fi.h.Mode).Perm() - - // Set setuid, setgid and sticky bits. - if fi.h.Mode&c_ISUID != 0 { - mode |= os.ModeSetuid - } - if fi.h.Mode&c_ISGID != 0 { - mode |= os.ModeSetgid - } - if fi.h.Mode&c_ISVTX != 0 { - mode |= os.ModeSticky - } - - // Set file mode bits; clear perm, setuid, setgid, and sticky bits. - switch m := os.FileMode(fi.h.Mode) &^ 07777; m { - case c_ISDIR: - mode |= os.ModeDir - case c_ISFIFO: - mode |= os.ModeNamedPipe - case c_ISLNK: - mode |= os.ModeSymlink - case c_ISBLK: - mode |= os.ModeDevice - case c_ISCHR: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case c_ISSOCK: - mode |= os.ModeSocket - } - - switch fi.h.Typeflag { - case TypeSymlink: - mode |= os.ModeSymlink - case TypeChar: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case TypeBlock: - mode |= os.ModeDevice - case TypeDir: - mode |= os.ModeDir - case TypeFifo: - mode |= os.ModeNamedPipe - } - - return mode -} - -// sysStat, if non-nil, populates h from system-dependent fields of fi. -var sysStat func(fi os.FileInfo, h *Header) error - -const ( - // Mode constants from the USTAR spec: - // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) - - // Common Unix mode constants; these are not defined in any common tar standard. - // Header.FileInfo understands these, but FileInfoHeader will never produce these. - c_ISDIR = 040000 // Directory - c_ISFIFO = 010000 // FIFO - c_ISREG = 0100000 // Regular file - c_ISLNK = 0120000 // Symbolic link - c_ISBLK = 060000 // Block special file - c_ISCHR = 020000 // Character special file - c_ISSOCK = 0140000 // Socket -) - -// FileInfoHeader creates a partially-populated Header from fi. -// If fi describes a symlink, FileInfoHeader records link as the link target. -// If fi describes a directory, a slash is appended to the name. -// -// Since os.FileInfo's Name method only returns the base name of -// the file it describes, it may be necessary to modify Header.Name -// to provide the full path name of the file. -func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { - if fi == nil { - return nil, errors.New("archive/tar: FileInfo is nil") - } - fm := fi.Mode() - h := &Header{ - Name: fi.Name(), - ModTime: fi.ModTime(), - Mode: int64(fm.Perm()), // or'd with c_IS* constants later - } - switch { - case fm.IsRegular(): - h.Typeflag = TypeReg - h.Size = fi.Size() - case fi.IsDir(): - h.Typeflag = TypeDir - h.Name += "/" - case fm&os.ModeSymlink != 0: - h.Typeflag = TypeSymlink - h.Linkname = link - case fm&os.ModeDevice != 0: - if fm&os.ModeCharDevice != 0 { - h.Typeflag = TypeChar - } else { - h.Typeflag = TypeBlock - } - case fm&os.ModeNamedPipe != 0: - h.Typeflag = TypeFifo - case fm&os.ModeSocket != 0: - return nil, fmt.Errorf("archive/tar: sockets not supported") - default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) - } - if fm&os.ModeSetuid != 0 { - h.Mode |= c_ISUID - } - if fm&os.ModeSetgid != 0 { - h.Mode |= c_ISGID - } - if fm&os.ModeSticky != 0 { - h.Mode |= c_ISVTX - } - // If possible, populate additional fields from OS-specific - // FileInfo fields. - if sys, ok := fi.Sys().(*Header); ok { - // This FileInfo came from a Header (not the OS). Use the - // original Header to populate all remaining fields. - h.Uid = sys.Uid - h.Gid = sys.Gid - h.Uname = sys.Uname - h.Gname = sys.Gname - h.AccessTime = sys.AccessTime - h.ChangeTime = sys.ChangeTime - if sys.Xattrs != nil { - h.Xattrs = make(map[string]string) - for k, v := range sys.Xattrs { - h.Xattrs[k] = v - } - } - if sys.Typeflag == TypeLink { - // hard link - h.Typeflag = TypeLink - h.Size = 0 - h.Linkname = sys.Linkname - } - if sys.PAXRecords != nil { - h.PAXRecords = make(map[string]string) - for k, v := range sys.PAXRecords { - h.PAXRecords[k] = v - } - } - } - if sysStat != nil { - return h, sysStat(fi, h) - } - return h, nil -} - -// isHeaderOnlyType checks if the given type flag is of the type that has no -// data section even if a size is specified. -func isHeaderOnlyType(flag byte) bool { - switch flag { - case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo: - return true - default: - return false - } -} - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/format.go b/vendor/github.com/vbatts/tar-split/archive/tar/format.go deleted file mode 100644 index 60977980c..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/format.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import "strings" - -// Format represents the tar archive format. -// -// The original tar format was introduced in Unix V7. -// Since then, there have been multiple competing formats attempting to -// standardize or extend the V7 format to overcome its limitations. -// The most common formats are the USTAR, PAX, and GNU formats, -// each with their own advantages and limitations. -// -// The following table captures the capabilities of each format: -// -// | USTAR | PAX | GNU -// ------------------+--------+-----------+---------- -// Name | 256B | unlimited | unlimited -// Linkname | 100B | unlimited | unlimited -// Size | uint33 | unlimited | uint89 -// Mode | uint21 | uint21 | uint57 -// Uid/Gid | uint21 | unlimited | uint57 -// Uname/Gname | 32B | unlimited | 32B -// ModTime | uint33 | unlimited | int89 -// AccessTime | n/a | unlimited | int89 -// ChangeTime | n/a | unlimited | int89 -// Devmajor/Devminor | uint21 | uint21 | uint57 -// ------------------+--------+-----------+---------- -// string encoding | ASCII | UTF-8 | binary -// sub-second times | no | yes | no -// sparse files | no | yes | yes -// -// The table's upper portion shows the Header fields, where each format reports -// the maximum number of bytes allowed for each string field and -// the integer type used to store each numeric field -// (where timestamps are stored as the number of seconds since the Unix epoch). -// -// The table's lower portion shows specialized features of each format, -// such as supported string encodings, support for sub-second timestamps, -// or support for sparse files. -// -// The Writer currently provides no support for sparse files. -type Format int - -// Constants to identify various tar formats. -const ( - // Deliberately hide the meaning of constants from public API. - _ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc... - - // FormatUnknown indicates that the format is unknown. - FormatUnknown - - // The format of the original Unix V7 tar tool prior to standardization. - formatV7 - - // FormatUSTAR represents the USTAR header format defined in POSIX.1-1988. - // - // While this format is compatible with most tar readers, - // the format has several limitations making it unsuitable for some usages. - // Most notably, it cannot support sparse files, files larger than 8GiB, - // filenames larger than 256 characters, and non-ASCII filenames. - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - FormatUSTAR - - // FormatPAX represents the PAX header format defined in POSIX.1-2001. - // - // PAX extends USTAR by writing a special file with Typeflag TypeXHeader - // preceding the original header. This file contains a set of key-value - // records, which are used to overcome USTAR's shortcomings, in addition to - // providing the ability to have sub-second resolution for timestamps. - // - // Some newer formats add their own extensions to PAX by defining their - // own keys and assigning certain semantic meaning to the associated values. - // For example, sparse file support in PAX is implemented using keys - // defined by the GNU manual (e.g., "GNU.sparse.map"). - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html - FormatPAX - - // FormatGNU represents the GNU header format. - // - // The GNU header format is older than the USTAR and PAX standards and - // is not compatible with them. The GNU format supports - // arbitrary file sizes, filenames of arbitrary encoding and length, - // sparse files, and other features. - // - // It is recommended that PAX be chosen over GNU unless the target - // application can only parse GNU formatted archives. - // - // Reference: - // https://www.gnu.org/software/tar/manual/html_node/Standard.html - FormatGNU - - // Schily's tar format, which is incompatible with USTAR. - // This does not cover STAR extensions to the PAX format; these fall under - // the PAX format. - formatSTAR - - formatMax -) - -func (f Format) has(f2 Format) bool { return f&f2 != 0 } -func (f *Format) mayBe(f2 Format) { *f |= f2 } -func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 } -func (f *Format) mustNotBe(f2 Format) { *f &^= f2 } - -var formatNames = map[Format]string{ - formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR", -} - -func (f Format) String() string { - var ss []string - for f2 := Format(1); f2 < formatMax; f2 <<= 1 { - if f.has(f2) { - ss = append(ss, formatNames[f2]) - } - } - switch len(ss) { - case 0: - return "" - case 1: - return ss[0] - default: - return "(" + strings.Join(ss, " | ") + ")" - } -} - -// Magics used to identify various formats. -const ( - magicGNU, versionGNU = "ustar ", " \x00" - magicUSTAR, versionUSTAR = "ustar\x00", "00" - trailerSTAR = "tar\x00" -) - -// Size constants from various tar specifications. -const ( - blockSize = 512 // Size of each block in a tar stream - nameSize = 100 // Max length of the name field in USTAR format - prefixSize = 155 // Max length of the prefix field in USTAR format - - // Max length of a special file (PAX header, GNU long name or link). - // This matches the limit used by libarchive. - maxSpecialFileSize = 1 << 20 -) - -// blockPadding computes the number of bytes needed to pad offset up to the -// nearest block edge where 0 <= n < blockSize. -func blockPadding(offset int64) (n int64) { - return -offset & (blockSize - 1) -} - -var zeroBlock block - -type block [blockSize]byte - -// Convert block to any number of formats. -func (b *block) V7() *headerV7 { return (*headerV7)(b) } -func (b *block) GNU() *headerGNU { return (*headerGNU)(b) } -func (b *block) STAR() *headerSTAR { return (*headerSTAR)(b) } -func (b *block) USTAR() *headerUSTAR { return (*headerUSTAR)(b) } -func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) } - -// GetFormat checks that the block is a valid tar header based on the checksum. -// It then attempts to guess the specific format based on magic values. -// If the checksum fails, then FormatUnknown is returned. -func (b *block) GetFormat() Format { - // Verify checksum. - var p parser - value := p.parseOctal(b.V7().Chksum()) - chksum1, chksum2 := b.ComputeChecksum() - if p.err != nil || (value != chksum1 && value != chksum2) { - return FormatUnknown - } - - // Guess the magic values. - magic := string(b.USTAR().Magic()) - version := string(b.USTAR().Version()) - trailer := string(b.STAR().Trailer()) - switch { - case magic == magicUSTAR && trailer == trailerSTAR: - return formatSTAR - case magic == magicUSTAR: - return FormatUSTAR | FormatPAX - case magic == magicGNU && version == versionGNU: - return FormatGNU - default: - return formatV7 - } -} - -// SetFormat writes the magic values necessary for specified format -// and then updates the checksum accordingly. -func (b *block) SetFormat(format Format) { - // Set the magic values. - switch { - case format.has(formatV7): - // Do nothing. - case format.has(FormatGNU): - copy(b.GNU().Magic(), magicGNU) - copy(b.GNU().Version(), versionGNU) - case format.has(formatSTAR): - copy(b.STAR().Magic(), magicUSTAR) - copy(b.STAR().Version(), versionUSTAR) - copy(b.STAR().Trailer(), trailerSTAR) - case format.has(FormatUSTAR | FormatPAX): - copy(b.USTAR().Magic(), magicUSTAR) - copy(b.USTAR().Version(), versionUSTAR) - default: - panic("invalid format") - } - - // Update checksum. - // This field is special in that it is terminated by a NULL then space. - var f formatter - field := b.V7().Chksum() - chksum, _ := b.ComputeChecksum() // Possible values are 256..128776 - f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143 - field[7] = ' ' -} - -// ComputeChecksum computes the checksum for the header block. -// POSIX specifies a sum of the unsigned byte values, but the Sun tar used -// signed byte values. -// We compute and return both. -func (b *block) ComputeChecksum() (unsigned, signed int64) { - for i, c := range b { - if 148 <= i && i < 156 { - c = ' ' // Treat the checksum field itself as all spaces. - } - unsigned += int64(c) - signed += int64(int8(c)) - } - return unsigned, signed -} - -// Reset clears the block with all zeros. -func (b *block) Reset() { - *b = block{} -} - -type headerV7 [blockSize]byte - -func (h *headerV7) Name() []byte { return h[000:][:100] } -func (h *headerV7) Mode() []byte { return h[100:][:8] } -func (h *headerV7) UID() []byte { return h[108:][:8] } -func (h *headerV7) GID() []byte { return h[116:][:8] } -func (h *headerV7) Size() []byte { return h[124:][:12] } -func (h *headerV7) ModTime() []byte { return h[136:][:12] } -func (h *headerV7) Chksum() []byte { return h[148:][:8] } -func (h *headerV7) TypeFlag() []byte { return h[156:][:1] } -func (h *headerV7) LinkName() []byte { return h[157:][:100] } - -type headerGNU [blockSize]byte - -func (h *headerGNU) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerGNU) Magic() []byte { return h[257:][:6] } -func (h *headerGNU) Version() []byte { return h[263:][:2] } -func (h *headerGNU) UserName() []byte { return h[265:][:32] } -func (h *headerGNU) GroupName() []byte { return h[297:][:32] } -func (h *headerGNU) DevMajor() []byte { return h[329:][:8] } -func (h *headerGNU) DevMinor() []byte { return h[337:][:8] } -func (h *headerGNU) AccessTime() []byte { return h[345:][:12] } -func (h *headerGNU) ChangeTime() []byte { return h[357:][:12] } -func (h *headerGNU) Sparse() sparseArray { return (sparseArray)(h[386:][:24*4+1]) } -func (h *headerGNU) RealSize() []byte { return h[483:][:12] } - -type headerSTAR [blockSize]byte - -func (h *headerSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerSTAR) Version() []byte { return h[263:][:2] } -func (h *headerSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerSTAR) Prefix() []byte { return h[345:][:131] } -func (h *headerSTAR) AccessTime() []byte { return h[476:][:12] } -func (h *headerSTAR) ChangeTime() []byte { return h[488:][:12] } -func (h *headerSTAR) Trailer() []byte { return h[508:][:4] } - -type headerUSTAR [blockSize]byte - -func (h *headerUSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerUSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerUSTAR) Version() []byte { return h[263:][:2] } -func (h *headerUSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerUSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerUSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerUSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] } - -type sparseArray []byte - -func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) } -func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] } -func (s sparseArray) MaxEntries() int { return len(s) / 24 } - -type sparseElem []byte - -func (s sparseElem) Offset() []byte { return s[00:][:12] } -func (s sparseElem) Length() []byte { return s[12:][:12] } diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go b/vendor/github.com/vbatts/tar-split/archive/tar/reader.go deleted file mode 100644 index a645c4160..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go +++ /dev/null @@ -1,944 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "io" - "strconv" - "strings" - "time" -) - -// Reader provides sequential access to the contents of a tar archive. -// Reader.Next advances to the next file in the archive (including the first), -// and then Reader can be treated as an io.Reader to access the file's data. -type Reader struct { - r io.Reader - pad int64 // Amount of padding (ignored) after current file entry - curr fileReader // Reader for current file entry - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Reader to - // ensure that this error is sticky. - err error - - RawAccounting bool // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this. - rawBytes *bytes.Buffer // last raw bits -} - -type fileReader interface { - io.Reader - fileState - - WriteTo(io.Writer) (int64, error) -} - -// RawBytes accesses the raw bytes of the archive, apart from the file payload itself. -// This includes the header and padding. -// -// # This call resets the current rawbytes buffer -// -// Only when RawAccounting is enabled, otherwise this returns nil -func (tr *Reader) RawBytes() []byte { - if !tr.RawAccounting { - return nil - } - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } - defer tr.rawBytes.Reset() // if we've read them, then flush them. - - return tr.rawBytes.Bytes() - -} - -// ExpectedPadding returns the number of bytes of padding expected after the last header returned by Next() -func (tr *Reader) ExpectedPadding() int64 { - return tr.pad -} - -// NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { - return &Reader{r: r, curr: ®FileReader{r, 0}} -} - -// Next advances to the next entry in the tar archive. -// The Header.Size determines how many bytes can be read for the next file. -// Any remaining data in the current file is automatically discarded. -// -// io.EOF is returned at the end of the input. -func (tr *Reader) Next() (*Header, error) { - if tr.err != nil { - return nil, tr.err - } - hdr, err := tr.next() - tr.err = err - return hdr, err -} - -func (tr *Reader) next() (*Header, error) { - var paxHdrs map[string]string - var gnuLongName, gnuLongLink string - - if tr.RawAccounting { - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } else { - tr.rawBytes.Reset() - } - } - - // Externally, Next iterates through the tar archive as if it is a series of - // files. Internally, the tar format often uses fake "files" to add meta - // data that describes the next file. These meta data "files" should not - // normally be visible to the outside. As such, this loop iterates through - // one or more "header files" until it finds a "normal file". - format := FormatUSTAR | FormatPAX | FormatGNU - for { - // Discard the remainder of the file and any padding. - if err := discard(tr, tr.curr.PhysicalRemaining()); err != nil { - return nil, err - } - n, err := tryReadFull(tr.r, tr.blk[:tr.pad]) - if err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(tr.blk[:n]) - } - tr.pad = 0 - - hdr, rawHdr, err := tr.readHeader() - if err != nil { - return nil, err - } - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - format.mayOnlyBe(hdr.Format) - - // Check for PAX/GNU special headers and files. - switch hdr.Typeflag { - case TypeXHeader, TypeXGlobalHeader: - format.mayOnlyBe(FormatPAX) - paxHdrs, err = parsePAX(tr) - if err != nil { - return nil, err - } - if hdr.Typeflag == TypeXGlobalHeader { - if err = mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - return &Header{ - Name: hdr.Name, - Typeflag: hdr.Typeflag, - Xattrs: hdr.Xattrs, - PAXRecords: hdr.PAXRecords, - Format: format, - }, nil - } - continue // This is a meta header affecting the next header - case TypeGNULongName, TypeGNULongLink: - format.mayOnlyBe(FormatGNU) - realname, err := readSpecialFile(tr) - if err != nil { - return nil, err - } - - if tr.RawAccounting { - tr.rawBytes.Write(realname) - } - - var p parser - switch hdr.Typeflag { - case TypeGNULongName: - gnuLongName = p.parseString(realname) - case TypeGNULongLink: - gnuLongLink = p.parseString(realname) - } - continue // This is a meta header affecting the next header - default: - // The old GNU sparse format is handled here since it is technically - // just a regular file with additional attributes. - - if err := mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - if gnuLongName != "" { - hdr.Name = gnuLongName - } - if gnuLongLink != "" { - hdr.Linkname = gnuLongLink - } - if hdr.Typeflag == TypeRegA { - if strings.HasSuffix(hdr.Name, "/") { - hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories - } else { - hdr.Typeflag = TypeReg - } - } - - // The extended headers may have updated the size. - // Thus, setup the regFileReader again after merging PAX headers. - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - - // Sparse formats rely on being able to read from the logical data - // section; there must be a preceding call to handleRegularFile. - if err := tr.handleSparseFile(hdr, rawHdr); err != nil { - return nil, err - } - - // Set the final guess at the format. - if format.has(FormatUSTAR) && format.has(FormatPAX) { - format.mayOnlyBe(FormatUSTAR) - } - hdr.Format = format - return hdr, nil // This is a file, so stop - } - } -} - -// handleRegularFile sets up the current file reader and padding such that it -// can only read the following logical data section. It will properly handle -// special headers that contain no data section. -func (tr *Reader) handleRegularFile(hdr *Header) error { - nb := hdr.Size - if isHeaderOnlyType(hdr.Typeflag) { - nb = 0 - } - if nb < 0 { - return ErrHeader - } - - tr.pad = blockPadding(nb) - tr.curr = ®FileReader{r: tr.r, nb: nb} - return nil -} - -// handleSparseFile checks if the current file is a sparse format of any type -// and sets the curr reader appropriately. -func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error { - var spd sparseDatas - var err error - if hdr.Typeflag == TypeGNUSparse { - spd, err = tr.readOldGNUSparseMap(hdr, rawHdr) - } else { - spd, err = tr.readGNUSparsePAXHeaders(hdr) - } - - // If sp is non-nil, then this is a sparse file. - // Note that it is possible for len(sp) == 0. - if err == nil && spd != nil { - if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) { - return ErrHeader - } - sph := invertSparseEntries(spd, hdr.Size) - tr.curr = &sparseFileReader{tr.curr, sph, 0} - } - return err -} - -// readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. -// If they are found, then this function reads the sparse map and returns it. -// This assumes that 0.0 headers have already been converted to 0.1 headers -// by the PAX header parsing logic. -func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) { - // Identify the version of GNU headers. - var is1x0 bool - major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor] - switch { - case major == "0" && (minor == "0" || minor == "1"): - is1x0 = false - case major == "1" && minor == "0": - is1x0 = true - case major != "" || minor != "": - return nil, nil // Unknown GNU sparse PAX version - case hdr.PAXRecords[paxGNUSparseMap] != "": - is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess - default: - return nil, nil // Not a PAX format GNU sparse file. - } - hdr.Format.mayOnlyBe(FormatPAX) - - // Update hdr from GNU sparse PAX headers. - if name := hdr.PAXRecords[paxGNUSparseName]; name != "" { - hdr.Name = name - } - size := hdr.PAXRecords[paxGNUSparseSize] - if size == "" { - size = hdr.PAXRecords[paxGNUSparseRealSize] - } - if size != "" { - n, err := strconv.ParseInt(size, 10, 64) - if err != nil { - return nil, ErrHeader - } - hdr.Size = n - } - - // Read the sparse map according to the appropriate format. - if is1x0 { - return readGNUSparseMap1x0(tr.curr) - } - return readGNUSparseMap0x1(hdr.PAXRecords) -} - -// mergePAX merges paxHdrs into hdr for all relevant fields of Header. -func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { - for k, v := range paxHdrs { - if v == "" { - continue // Keep the original USTAR value - } - var id64 int64 - switch k { - case paxPath: - hdr.Name = v - case paxLinkpath: - hdr.Linkname = v - case paxUname: - hdr.Uname = v - case paxGname: - hdr.Gname = v - case paxUid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Uid = int(id64) // Integer overflow possible - case paxGid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Gid = int(id64) // Integer overflow possible - case paxAtime: - hdr.AccessTime, err = parsePAXTime(v) - case paxMtime: - hdr.ModTime, err = parsePAXTime(v) - case paxCtime: - hdr.ChangeTime, err = parsePAXTime(v) - case paxSize: - hdr.Size, err = strconv.ParseInt(v, 10, 64) - default: - if strings.HasPrefix(k, paxSchilyXattr) { - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) - } - hdr.Xattrs[k[len(paxSchilyXattr):]] = v - } - } - if err != nil { - return ErrHeader - } - } - hdr.PAXRecords = paxHdrs - return nil -} - -// parsePAX parses PAX headers. -// If an extended header (type 'x') is invalid, ErrHeader is returned -func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := readSpecialFile(r) - if err != nil { - return nil, err - } - // leaving this function for io.Reader makes it more testable - if tr, ok := r.(*Reader); ok && tr.RawAccounting { - if _, err = tr.rawBytes.Write(buf); err != nil { - return nil, err - } - } - sbuf := string(buf) - - // For GNU PAX sparse format 0.0 support. - // This function transforms the sparse format 0.0 headers into format 0.1 - // headers since 0.0 headers were not PAX compliant. - var sparseMap []string - - paxHdrs := make(map[string]string) - for len(sbuf) > 0 { - key, value, residual, err := parsePAXRecord(sbuf) - if err != nil { - return nil, ErrHeader - } - sbuf = residual - - switch key { - case paxGNUSparseOffset, paxGNUSparseNumBytes: - // Validate sparse header order and value. - if (len(sparseMap)%2 == 0 && key != paxGNUSparseOffset) || - (len(sparseMap)%2 == 1 && key != paxGNUSparseNumBytes) || - strings.Contains(value, ",") { - return nil, ErrHeader - } - sparseMap = append(sparseMap, value) - default: - paxHdrs[key] = value - } - } - if len(sparseMap) > 0 { - paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") - } - return paxHdrs, nil -} - -// readHeader reads the next block header and assumes that the underlying reader -// is already aligned to a block boundary. It returns the raw block of the -// header in case further processing is required. -// -// The err will be set to io.EOF only when one of the following occurs: -// - Exactly 0 bytes are read and EOF is hit. -// - Exactly 1 block of zeros is read and EOF is hit. -// - At least 2 blocks of zeros are read. -func (tr *Reader) readHeader() (*Header, *block, error) { - // Two blocks of zero bytes marks the end of the archive. - n, err := io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 0 bytes read - } - - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - n, err = io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 1 block of zeros read - } - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - return nil, nil, io.EOF // normal EOF; exactly 2 block of zeros read - } - return nil, nil, ErrHeader // Zero block and then non-zero block - } - - // Verify the header matches a known format. - format := tr.blk.GetFormat() - if format == FormatUnknown { - return nil, nil, ErrHeader - } - - var p parser - hdr := new(Header) - - // Unpack the V7 header. - v7 := tr.blk.V7() - hdr.Typeflag = v7.TypeFlag()[0] - hdr.Name = p.parseString(v7.Name()) - hdr.Linkname = p.parseString(v7.LinkName()) - hdr.Size = p.parseNumeric(v7.Size()) - hdr.Mode = p.parseNumeric(v7.Mode()) - hdr.Uid = int(p.parseNumeric(v7.UID())) - hdr.Gid = int(p.parseNumeric(v7.GID())) - hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0) - - // Unpack format specific fields. - if format > formatV7 { - ustar := tr.blk.USTAR() - hdr.Uname = p.parseString(ustar.UserName()) - hdr.Gname = p.parseString(ustar.GroupName()) - hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) - hdr.Devminor = p.parseNumeric(ustar.DevMinor()) - - var prefix string - switch { - case format.has(FormatUSTAR | FormatPAX): - hdr.Format = format - ustar := tr.blk.USTAR() - prefix = p.parseString(ustar.Prefix()) - - // For Format detection, check if block is properly formatted since - // the parser is more liberal than what USTAR actually permits. - notASCII := func(r rune) bool { return r >= 0x80 } - if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 { - hdr.Format = FormatUnknown // Non-ASCII characters in block. - } - nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 } - if !(nul(v7.Size()) && nul(v7.Mode()) && nul(v7.UID()) && nul(v7.GID()) && - nul(v7.ModTime()) && nul(ustar.DevMajor()) && nul(ustar.DevMinor())) { - hdr.Format = FormatUnknown // Numeric fields must end in NUL - } - case format.has(formatSTAR): - star := tr.blk.STAR() - prefix = p.parseString(star.Prefix()) - hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0) - hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0) - case format.has(FormatGNU): - hdr.Format = format - var p2 parser - gnu := tr.blk.GNU() - if b := gnu.AccessTime(); b[0] != 0 { - hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0) - } - if b := gnu.ChangeTime(); b[0] != 0 { - hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0) - } - - // Prior to Go1.8, the Writer had a bug where it would output - // an invalid tar file in certain rare situations because the logic - // incorrectly believed that the old GNU format had a prefix field. - // This is wrong and leads to an output file that mangles the - // atime and ctime fields, which are often left unused. - // - // In order to continue reading tar files created by former, buggy - // versions of Go, we skeptically parse the atime and ctime fields. - // If we are unable to parse them and the prefix field looks like - // an ASCII string, then we fallback on the pre-Go1.8 behavior - // of treating these fields as the USTAR prefix field. - // - // Note that this will not use the fallback logic for all possible - // files generated by a pre-Go1.8 toolchain. If the generated file - // happened to have a prefix field that parses as valid - // atime and ctime fields (e.g., when they are valid octal strings), - // then it is impossible to distinguish between an valid GNU file - // and an invalid pre-Go1.8 file. - // - // See https://golang.org/issues/12594 - // See https://golang.org/issues/21005 - if p2.err != nil { - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - ustar := tr.blk.USTAR() - if s := p.parseString(ustar.Prefix()); isASCII(s) { - prefix = s - } - hdr.Format = FormatUnknown // Buggy file is not GNU - } - } - if len(prefix) > 0 { - hdr.Name = prefix + "/" + hdr.Name - } - } - return hdr, &tr.blk, p.err -} - -// readOldGNUSparseMap reads the sparse map from the old GNU sparse format. -// The sparse map is stored in the tar header if it's small enough. -// If it's larger than four entries, then one or more extension headers are used -// to store the rest of the sparse map. -// -// The Header.Size does not reflect the size of any extended headers used. -// Thus, this function will read from the raw io.Reader to fetch extra headers. -// This method mutates blk in the process. -func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) { - // Make sure that the input format is GNU. - // Unfortunately, the STAR format also has a sparse header format that uses - // the same type flag but has a completely different layout. - if blk.GetFormat() != FormatGNU { - return nil, ErrHeader - } - hdr.Format.mayOnlyBe(FormatGNU) - - var p parser - hdr.Size = p.parseNumeric(blk.GNU().RealSize()) - if p.err != nil { - return nil, p.err - } - s := blk.GNU().Sparse() - spd := make(sparseDatas, 0, s.MaxEntries()) - for { - for i := 0; i < s.MaxEntries(); i++ { - // This termination condition is identical to GNU and BSD tar. - if s.Entry(i).Offset()[0] == 0x00 { - break // Don't return, need to process extended headers (even if empty) - } - offset := p.parseNumeric(s.Entry(i).Offset()) - length := p.parseNumeric(s.Entry(i).Length()) - if p.err != nil { - return nil, p.err - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - - if s.IsExtended()[0] > 0 { - // There are more entries. Read an extension header and parse its entries. - if _, err := mustReadFull(tr.r, blk[:]); err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(blk[:]) - } - s = blk.Sparse() - continue - } - return spd, nil // Done - } -} - -// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format -// version 1.0. The format of the sparse map consists of a series of -// newline-terminated numeric fields. The first field is the number of entries -// and is always present. Following this are the entries, consisting of two -// fields (offset, length). This function must stop reading at the end -// boundary of the block containing the last newline. -// -// Note that the GNU manual says that numeric values should be encoded in octal -// format. However, the GNU tar utility itself outputs these values in decimal. -// As such, this library treats values as being encoded in decimal. -func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) { - var ( - cntNewline int64 - buf bytes.Buffer - blk block - totalSize int - ) - - // feedTokens copies data in blocks from r into buf until there are - // at least cnt newlines in buf. It will not read more blocks than needed. - feedTokens := func(n int64) error { - for cntNewline < n { - totalSize += len(blk) - if totalSize > maxSpecialFileSize { - return errSparseTooLong - } - if _, err := mustReadFull(r, blk[:]); err != nil { - return err - } - buf.Write(blk[:]) - for _, c := range blk { - if c == '\n' { - cntNewline++ - } - } - } - return nil - } - - // nextToken gets the next token delimited by a newline. This assumes that - // at least one newline exists in the buffer. - nextToken := func() string { - cntNewline-- - tok, _ := buf.ReadString('\n') - return strings.TrimRight(tok, "\n") - } - - // Parse for the number of entries. - // Use integer overflow resistant math to check this. - if err := feedTokens(1); err != nil { - return nil, err - } - numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // Parse for all member entries. - // numEntries is trusted after this since feedTokens limits the number of - // tokens based on maxSpecialFileSize. - if err := feedTokens(2 * numEntries); err != nil { - return nil, err - } - spd := make(sparseDatas, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - offset, err1 := strconv.ParseInt(nextToken(), 10, 64) - length, err2 := strconv.ParseInt(nextToken(), 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - return spd, nil -} - -// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format -// version 0.1. The sparse map is stored in the PAX headers. -func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) { - // Get number of entries. - // Use integer overflow resistant math to check this. - numEntriesStr := paxHdrs[paxGNUSparseNumBlocks] - numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // There should be two numbers in sparseMap for each entry. - sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",") - if len(sparseMap) == 1 && sparseMap[0] == "" { - sparseMap = sparseMap[:0] - } - if int64(len(sparseMap)) != 2*numEntries { - return nil, ErrHeader - } - - // Loop through the entries in the sparse map. - // numEntries is trusted now. - spd := make(sparseDatas, 0, numEntries) - for len(sparseMap) >= 2 { - offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64) - length, err2 := strconv.ParseInt(sparseMap[1], 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - sparseMap = sparseMap[2:] - } - return spd, nil -} - -// Read reads from the current file in the tar archive. -// It returns (0, io.EOF) when it reaches the end of that file, -// until Next is called to advance to the next file. -// -// If the current file is sparse, then the regions marked as a hole -// are read back as NUL-bytes. -// -// Calling Read on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what -// the Header.Size claims. -func (tr *Reader) Read(b []byte) (int, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.Read(b) - if err != nil && err != io.EOF { - tr.err = err - } - return n, err -} - -// writeTo writes the content of the current file to w. -// The bytes written matches the number of remaining bytes in the current file. -// -// If the current file is sparse and w is an io.WriteSeeker, -// then writeTo uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are filled with NULs. -// This always writes the last byte to ensure w is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tr *Reader) writeTo(w io.Writer) (int64, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.WriteTo(w) - if err != nil { - tr.err = err - } - return n, err -} - -// regFileReader is a fileReader for reading data from a regular file entry. -type regFileReader struct { - r io.Reader // Underlying Reader - nb int64 // Number of remaining bytes to read -} - -func (fr *regFileReader) Read(b []byte) (n int, err error) { - if int64(len(b)) > fr.nb { - b = b[:fr.nb] - } - if len(b) > 0 { - n, err = fr.r.Read(b) - fr.nb -= int64(n) - } - switch { - case err == io.EOF && fr.nb > 0: - return n, io.ErrUnexpectedEOF - case err == nil && fr.nb == 0: - return n, io.EOF - default: - return n, err - } -} - -func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { - return io.Copy(w, struct{ io.Reader }{fr}) -} - -func (fr regFileReader) LogicalRemaining() int64 { - return fr.nb -} - -func (fr regFileReader) PhysicalRemaining() int64 { - return fr.nb -} - -// sparseFileReader is a fileReader for reading data from a sparse file entry. -type sparseFileReader struct { - fr fileReader // Underlying fileReader - sp sparseHoles // Normalized list of sparse holes - pos int64 // Current position in sparse file -} - -func (sr *sparseFileReader) Read(b []byte) (n int, err error) { - finished := int64(len(b)) >= sr.LogicalRemaining() - if finished { - b = b[:sr.LogicalRemaining()] - } - - b0 := b - endPos := sr.pos + int64(len(b)) - for endPos > sr.pos && err == nil { - var nf int // Bytes read in fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - bf := b[:min(int64(len(b)), holeStart-sr.pos)] - nf, err = tryReadFull(sr.fr, bf) - } else { // In a hole fragment - bf := b[:min(int64(len(b)), holeEnd-sr.pos)] - nf, err = tryReadFull(zeroReader{}, bf) - } - b = b[nf:] - sr.pos += int64(nf) - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - case finished: - return n, io.EOF - default: - return n, nil - } -} - -func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { - ws, ok := w.(io.WriteSeeker) - if ok { - if _, err := ws.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(w, struct{ io.Reader }{sr}) - } - - var writeLastByte bool - pos0 := sr.pos - for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil { - var nf int64 // Size of fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - nf = holeStart - sr.pos - nf, err = io.CopyN(ws, sr.fr, nf) - } else { // In a hole fragment - nf = holeEnd - sr.pos - if sr.PhysicalRemaining() == 0 { - writeLastByte = true - nf-- - } - _, err = ws.Seek(nf, io.SeekCurrent) - } - sr.pos += nf - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // write a single byte to ensure the file is the right size. - if writeLastByte && err == nil { - _, err = ws.Write([]byte{0}) - sr.pos++ - } - - n = sr.pos - pos0 - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - default: - return n, nil - } -} - -func (sr sparseFileReader) LogicalRemaining() int64 { - return sr.sp[len(sr.sp)-1].endOffset() - sr.pos -} -func (sr sparseFileReader) PhysicalRemaining() int64 { - return sr.fr.PhysicalRemaining() -} - -type zeroReader struct{} - -func (zeroReader) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - -// mustReadFull is like io.ReadFull except it returns -// io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read. -func mustReadFull(r io.Reader, b []byte) (int, error) { - n, err := tryReadFull(r, b) - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return n, err -} - -// tryReadFull is like io.ReadFull except it returns -// io.EOF when it is hit before len(b) bytes are read. -func tryReadFull(r io.Reader, b []byte) (n int, err error) { - for len(b) > n && err == nil { - var nn int - nn, err = r.Read(b[n:]) - n += nn - } - if len(b) == n && err == io.EOF { - err = nil - } - return n, err -} - -// readSpecialFile is like io.ReadAll except it returns -// ErrFieldTooLong if more than maxSpecialFileSize is read. -func readSpecialFile(r io.Reader) ([]byte, error) { - buf, err := io.ReadAll(io.LimitReader(r, maxSpecialFileSize+1)) - if len(buf) > maxSpecialFileSize { - return nil, ErrFieldTooLong - } - return buf, err -} - -// discard skips n bytes in r, reporting an error if unable to do so. -func discard(tr *Reader, n int64) error { - var seekSkipped, copySkipped int64 - var err error - r := tr.r - if tr.RawAccounting { - - copySkipped, err = io.CopyN(tr.rawBytes, tr.r, n) - goto out - } - - // If possible, Seek to the last byte before the end of the data section. - // Do this because Seek is often lazy about reporting errors; this will mask - // the fact that the stream may be truncated. We can rely on the - // io.CopyN done shortly afterwards to trigger any IO errors. - if sr, ok := r.(io.Seeker); ok && n > 1 { - // Not all io.Seeker can actually Seek. For example, os.Stdin implements - // io.Seeker, but calling Seek always returns an error and performs - // no action. Thus, we try an innocent seek to the current position - // to see if Seek is really supported. - pos1, err := sr.Seek(0, io.SeekCurrent) - if pos1 >= 0 && err == nil { - // Seek seems supported, so perform the real Seek. - pos2, err := sr.Seek(n-1, io.SeekCurrent) - if pos2 < 0 || err != nil { - return err - } - seekSkipped = pos2 - pos1 - } - } - - copySkipped, err = io.CopyN(io.Discard, r, n-seekSkipped) -out: - if err == io.EOF && seekSkipped+copySkipped < n { - err = io.ErrUnexpectedEOF - } - return err -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go deleted file mode 100644 index cf9cc79c5..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux dragonfly openbsd solaris - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atim.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctim.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go deleted file mode 100644 index 6f17dbe30..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin freebsd netbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atimespec.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctimespec.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go deleted file mode 100644 index 868105f33..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux darwin dragonfly freebsd openbsd netbsd solaris - -package tar - -import ( - "os" - "os/user" - "runtime" - "strconv" - "sync" - "syscall" -) - -func init() { - sysStat = statUnix -} - -// userMap and groupMap caches UID and GID lookups for performance reasons. -// The downside is that renaming uname or gname by the OS never takes effect. -var userMap, groupMap sync.Map // map[int]string - -func statUnix(fi os.FileInfo, h *Header) error { - sys, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return nil - } - h.Uid = int(sys.Uid) - h.Gid = int(sys.Gid) - - // Best effort at populating Uname and Gname. - // The os/user functions may fail for any number of reasons - // (not implemented on that platform, cgo not enabled, etc). - if u, ok := userMap.Load(h.Uid); ok { - h.Uname = u.(string) - } else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil { - h.Uname = u.Username - userMap.Store(h.Uid, h.Uname) - } - if g, ok := groupMap.Load(h.Gid); ok { - h.Gname = g.(string) - } else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil { - h.Gname = g.Name - groupMap.Store(h.Gid, h.Gname) - } - - h.AccessTime = statAtime(sys) - h.ChangeTime = statCtime(sys) - - // Best effort at populating Devmajor and Devminor. - if h.Typeflag == TypeChar || h.Typeflag == TypeBlock { - dev := uint64(sys.Rdev) // May be int32 or uint32 - switch runtime.GOOS { - case "linux": - // Copied from golang.org/x/sys/unix/dev_linux.go. - major := uint32((dev & 0x00000000000fff00) >> 8) - major |= uint32((dev & 0xfffff00000000000) >> 32) - minor := uint32((dev & 0x00000000000000ff) >> 0) - minor |= uint32((dev & 0x00000ffffff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "darwin": - // Copied from golang.org/x/sys/unix/dev_darwin.go. - major := uint32((dev >> 24) & 0xff) - minor := uint32(dev & 0xffffff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "dragonfly": - // Copied from golang.org/x/sys/unix/dev_dragonfly.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "freebsd": - // Copied from golang.org/x/sys/unix/dev_freebsd.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "netbsd": - // Copied from golang.org/x/sys/unix/dev_netbsd.go. - major := uint32((dev & 0x000fff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xfff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "openbsd": - // Copied from golang.org/x/sys/unix/dev_openbsd.go. - major := uint32((dev & 0x0000ff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xffff0000) >> 8) - h.Devmajor, h.Devminor = int64(major), int64(minor) - default: - // TODO: Implement solaris (see https://golang.org/issue/8106) - } - } - return nil -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go b/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go deleted file mode 100644 index d144485a4..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go +++ /dev/null @@ -1,326 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "time" -) - -// hasNUL reports whether the NUL character exists within s. -func hasNUL(s string) bool { - return strings.IndexByte(s, 0) >= 0 -} - -// isASCII reports whether the input is an ASCII C-style string. -func isASCII(s string) bool { - for _, c := range s { - if c >= 0x80 || c == 0x00 { - return false - } - } - return true -} - -// toASCII converts the input to an ASCII C-style string. -// This a best effort conversion, so invalid characters are dropped. -func toASCII(s string) string { - if isASCII(s) { - return s - } - b := make([]byte, 0, len(s)) - for _, c := range s { - if c < 0x80 && c != 0x00 { - b = append(b, byte(c)) - } - } - return string(b) -} - -type parser struct { - err error // Last error seen -} - -type formatter struct { - err error // Last error seen -} - -// parseString parses bytes as a NUL-terminated C-style string. -// If a NUL byte is not found then the whole slice is returned as a string. -func (*parser) parseString(b []byte) string { - if i := bytes.IndexByte(b, 0); i >= 0 { - return string(b[:i]) - } - return string(b) -} - -// formatString copies s into b, NUL-terminating if possible. -func (f *formatter) formatString(b []byte, s string) { - if len(s) > len(b) { - f.err = ErrFieldTooLong - } - copy(b, s) - if len(s) < len(b) { - b[len(s)] = 0 - } - - // Some buggy readers treat regular files with a trailing slash - // in the V7 path field as a directory even though the full path - // recorded elsewhere (e.g., via PAX record) contains no trailing slash. - if len(s) > len(b) && b[len(b)-1] == '/' { - n := len(strings.TrimRight(s[:len(b)], "/")) - b[n] = 0 // Replace trailing slash with NUL terminator - } -} - -// fitsInBase256 reports whether x can be encoded into n bytes using base-256 -// encoding. Unlike octal encoding, base-256 encoding does not require that the -// string ends with a NUL character. Thus, all n bytes are available for output. -// -// If operating in binary mode, this assumes strict GNU binary mode; which means -// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is -// equivalent to the sign bit in two's complement form. -func fitsInBase256(n int, x int64) bool { - binBits := uint(n-1) * 8 - return n >= 9 || (x >= -1< 0 && b[0]&0x80 != 0 { - // Handling negative numbers relies on the following identity: - // -a-1 == ^a - // - // If the number is negative, we use an inversion mask to invert the - // data bytes and treat the value as an unsigned number. - var inv byte // 0x00 if positive or zero, 0xff if negative - if b[0]&0x40 != 0 { - inv = 0xff - } - - var x uint64 - for i, c := range b { - c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing - if i == 0 { - c &= 0x7f // Ignore signal bit in first byte - } - if (x >> 56) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - x = x<<8 | uint64(c) - } - if (x >> 63) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - if inv == 0xff { - return ^int64(x) - } - return int64(x) - } - - // Normal case is base-8 (octal) format. - return p.parseOctal(b) -} - -// formatNumeric encodes x into b using base-8 (octal) encoding if possible. -// Otherwise it will attempt to use base-256 (binary) encoding. -func (f *formatter) formatNumeric(b []byte, x int64) { - if fitsInOctal(len(b), x) { - f.formatOctal(b, x) - return - } - - if fitsInBase256(len(b), x) { - for i := len(b) - 1; i >= 0; i-- { - b[i] = byte(x) - x >>= 8 - } - b[0] |= 0x80 // Highest bit indicates binary format - return - } - - f.formatOctal(b, 0) // Last resort, just write zero - f.err = ErrFieldTooLong -} - -func (p *parser) parseOctal(b []byte) int64 { - // Because unused fields are filled with NULs, we need - // to skip leading NULs. Fields may also be padded with - // spaces or NULs. - // So we remove leading and trailing NULs and spaces to - // be sure. - b = bytes.Trim(b, " \x00") - - if len(b) == 0 { - return 0 - } - x, perr := strconv.ParseUint(p.parseString(b), 8, 64) - if perr != nil { - p.err = ErrHeader - } - return int64(x) -} - -func (f *formatter) formatOctal(b []byte, x int64) { - if !fitsInOctal(len(b), x) { - x = 0 // Last resort, just write zero - f.err = ErrFieldTooLong - } - - s := strconv.FormatInt(x, 8) - // Add leading zeros, but leave room for a NUL. - if n := len(b) - len(s) - 1; n > 0 { - s = strings.Repeat("0", n) + s - } - f.formatString(b, s) -} - -// fitsInOctal reports whether the integer x fits in a field n-bytes long -// using octal encoding with the appropriate NUL terminator. -func fitsInOctal(n int, x int64) bool { - octBits := uint(n-1) * 3 - return x >= 0 && (n >= 22 || x < 1<= 0 { - ss, sn = s[:pos], s[pos+1:] - } - - // Parse the seconds. - secs, err := strconv.ParseInt(ss, 10, 64) - if err != nil { - return time.Time{}, ErrHeader - } - if len(sn) == 0 { - return time.Unix(secs, 0), nil // No sub-second values - } - - // Parse the nanoseconds. - if strings.Trim(sn, "0123456789") != "" { - return time.Time{}, ErrHeader - } - if len(sn) < maxNanoSecondDigits { - sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad - } else { - sn = sn[:maxNanoSecondDigits] // Right truncate - } - nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed - if len(ss) > 0 && ss[0] == '-' { - return time.Unix(secs, -1*nsecs), nil // Negative correction - } - return time.Unix(secs, nsecs), nil -} - -// formatPAXTime converts ts into a time of the form %d.%d as described in the -// PAX specification. This function is capable of negative timestamps. -func formatPAXTime(ts time.Time) (s string) { - secs, nsecs := ts.Unix(), ts.Nanosecond() - if nsecs == 0 { - return strconv.FormatInt(secs, 10) - } - - // If seconds is negative, then perform correction. - sign := "" - if secs < 0 { - sign = "-" // Remember sign - secs = -(secs + 1) // Add a second to secs - nsecs = -(nsecs - 1E9) // Take that second away from nsecs - } - return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0") -} - -// parsePAXRecord parses the input PAX record string into a key-value pair. -// If parsing is successful, it will slice off the currently read record and -// return the remainder as r. -func parsePAXRecord(s string) (k, v, r string, err error) { - // The size field ends at the first space. - sp := strings.IndexByte(s, ' ') - if sp == -1 { - return "", "", s, ErrHeader - } - - // Parse the first token as a decimal integer. - n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int - if perr != nil || n < 5 || int64(len(s)) < n { - return "", "", s, ErrHeader - } - - // Extract everything between the space and the final newline. - rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:] - if nl != "\n" { - return "", "", s, ErrHeader - } - - // The first equals separates the key from the value. - eq := strings.IndexByte(rec, '=') - if eq == -1 { - return "", "", s, ErrHeader - } - k, v = rec[:eq], rec[eq+1:] - - if !validPAXRecord(k, v) { - return "", "", s, ErrHeader - } - return k, v, rem, nil -} - -// formatPAXRecord formats a single PAX record, prefixing it with the -// appropriate length. -func formatPAXRecord(k, v string) (string, error) { - if !validPAXRecord(k, v) { - return "", ErrHeader - } - - const padding = 3 // Extra padding for ' ', '=', and '\n' - size := len(k) + len(v) + padding - size += len(strconv.Itoa(size)) - record := strconv.Itoa(size) + " " + k + "=" + v + "\n" - - // Final adjustment if adding size field increased the record size. - if len(record) != size { - size = len(record) - record = strconv.Itoa(size) + " " + k + "=" + v + "\n" - } - return record, nil -} - -// validPAXRecord reports whether the key-value pair is valid where each -// record is formatted as: -// "%d %s=%s\n" % (size, key, value) -// -// Keys and values should be UTF-8, but the number of bad writers out there -// forces us to be a more liberal. -// Thus, we only reject all keys with NUL, and only reject NULs in values -// for the PAX version of the USTAR string fields. -// The key must not contain an '=' character. -func validPAXRecord(k, v string) bool { - if k == "" || strings.IndexByte(k, '=') >= 0 { - return false - } - switch k { - case paxPath, paxLinkpath, paxUname, paxGname: - return !hasNUL(v) - default: - return !hasNUL(k) - } -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go b/vendor/github.com/vbatts/tar-split/archive/tar/writer.go deleted file mode 100644 index 893eac00a..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go +++ /dev/null @@ -1,656 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "fmt" - "io" - "path" - "sort" - "strings" - "time" -) - -// Writer provides sequential writing of a tar archive. -// Write.WriteHeader begins a new file with the provided Header, -// and then Writer can be treated as an io.Writer to supply that file's data. -type Writer struct { - w io.Writer - pad int64 // Amount of padding to write after current file entry - curr fileWriter // Writer for current file entry - hdr Header // Shallow copy of Header that is safe for mutations - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Writer to - // ensure that this error is sticky. - err error -} - -// NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { - return &Writer{w: w, curr: ®FileWriter{w, 0}} -} - -type fileWriter interface { - io.Writer - fileState - - ReadFrom(io.Reader) (int64, error) -} - -// Flush finishes writing the current file's block padding. -// The current file must be fully written before Flush can be called. -// -// This is unnecessary as the next call to WriteHeader or Close -// will implicitly flush out the file's padding. -func (tw *Writer) Flush() error { - if tw.err != nil { - return tw.err - } - if nb := tw.curr.LogicalRemaining(); nb > 0 { - return fmt.Errorf("archive/tar: missed writing %d bytes", nb) - } - if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil { - return tw.err - } - tw.pad = 0 - return nil -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// The Header.Size determines how many bytes can be written for the next file. -// If the current file is not fully written, then this returns an error. -// This implicitly flushes any padding necessary before writing the header. -func (tw *Writer) WriteHeader(hdr *Header) error { - if err := tw.Flush(); err != nil { - return err - } - tw.hdr = *hdr // Shallow copy of Header - - // Avoid usage of the legacy TypeRegA flag, and automatically promote - // it to use TypeReg or TypeDir. - if tw.hdr.Typeflag == TypeRegA { - if strings.HasSuffix(tw.hdr.Name, "/") { - tw.hdr.Typeflag = TypeDir - } else { - tw.hdr.Typeflag = TypeReg - } - } - - // Round ModTime and ignore AccessTime and ChangeTime unless - // the format is explicitly chosen. - // This ensures nominal usage of WriteHeader (without specifying the format) - // does not always result in the PAX format being chosen, which - // causes a 1KiB increase to every header. - if tw.hdr.Format == FormatUnknown { - tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second) - tw.hdr.AccessTime = time.Time{} - tw.hdr.ChangeTime = time.Time{} - } - - allowedFormats, paxHdrs, err := tw.hdr.allowedFormats() - switch { - case allowedFormats.has(FormatUSTAR): - tw.err = tw.writeUSTARHeader(&tw.hdr) - return tw.err - case allowedFormats.has(FormatPAX): - tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs) - return tw.err - case allowedFormats.has(FormatGNU): - tw.err = tw.writeGNUHeader(&tw.hdr) - return tw.err - default: - return err // Non-fatal error - } -} - -func (tw *Writer) writeUSTARHeader(hdr *Header) error { - // Check if we can use USTAR prefix/suffix splitting. - var namePrefix string - if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok { - namePrefix, hdr.Name = prefix, suffix - } - - // Pack the main header. - var f formatter - blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal) - f.formatString(blk.USTAR().Prefix(), namePrefix) - blk.SetFormat(FormatUSTAR) - if f.err != nil { - return f.err // Should never happen since header is validated - } - return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag) -} - -func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { - realName, realSize := hdr.Name, hdr.Size - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Handle sparse files. - var spd sparseDatas - var spb []byte - if len(hdr.SparseHoles) > 0 { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - hdr.Size = 0 // Replace with encoded size - spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n') - for _, s := range spd { - hdr.Size += s.Length - spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n') - spb = append(strconv.AppendInt(spb, s.Length, 10), '\n') - } - pad := blockPadding(int64(len(spb))) - spb = append(spb, zeroBlock[:pad]...) - hdr.Size += int64(len(spb)) // Accounts for encoded sparse map - - // Add and modify appropriate PAX records. - dir, file := path.Split(realName) - hdr.Name = path.Join(dir, "GNUSparseFile.0", file) - paxHdrs[paxGNUSparseMajor] = "1" - paxHdrs[paxGNUSparseMinor] = "0" - paxHdrs[paxGNUSparseName] = realName - paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10) - paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10) - delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName - } - */ - _ = realSize - - // Write PAX records to the output. - isGlobal := hdr.Typeflag == TypeXGlobalHeader - if len(paxHdrs) > 0 || isGlobal { - // Sort keys for deterministic ordering. - var keys []string - for k := range paxHdrs { - keys = append(keys, k) - } - sort.Strings(keys) - - // Write each record to a buffer. - var buf strings.Builder - for _, k := range keys { - rec, err := formatPAXRecord(k, paxHdrs[k]) - if err != nil { - return err - } - buf.WriteString(rec) - } - - // Write the extended header file. - var name string - var flag byte - if isGlobal { - name = realName - if name == "" { - name = "GlobalHead.0.0" - } - flag = TypeXGlobalHeader - } else { - dir, file := path.Split(realName) - name = path.Join(dir, "PaxHeaders.0", file) - flag = TypeXHeader - } - data := buf.String() - if len(data) > maxSpecialFileSize { - return ErrFieldTooLong - } - if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal { - return err // Global headers return here - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) } - blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal) - blk.SetFormat(FormatPAX) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Write the sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.curr since the sparse map is accounted for in hdr.Size. - if _, err := tw.curr.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - */ - return nil -} - -func (tw *Writer) writeGNUHeader(hdr *Header) error { - // Use long-link files if Name or Linkname exceeds the field size. - const longName = "././@LongLink" - if len(hdr.Name) > nameSize { - data := hdr.Name + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil { - return err - } - } - if len(hdr.Linkname) > nameSize { - data := hdr.Linkname + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil { - return err - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - var spd sparseDatas - var spb []byte - blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric) - if !hdr.AccessTime.IsZero() { - f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix()) - } - if !hdr.ChangeTime.IsZero() { - f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix()) - } - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - if hdr.Typeflag == TypeGNUSparse { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas { - for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ { - f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset) - f.formatNumeric(sa.Entry(i).Length(), sp[0].Length) - sp = sp[1:] - } - if len(sp) > 0 { - sa.IsExtended()[0] = 1 - } - return sp - } - sp2 := formatSPD(spd, blk.GNU().Sparse()) - for len(sp2) > 0 { - var spHdr block - sp2 = formatSPD(sp2, spHdr.Sparse()) - spb = append(spb, spHdr[:]...) - } - - // Update size fields in the header block. - realSize := hdr.Size - hdr.Size = 0 // Encoded size; does not account for encoded sparse map - for _, s := range spd { - hdr.Size += s.Length - } - copy(blk.V7().Size(), zeroBlock[:]) // Reset field - f.formatNumeric(blk.V7().Size(), hdr.Size) - f.formatNumeric(blk.GNU().RealSize(), realSize) - } - */ - blk.SetFormat(FormatGNU) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // Write the extended sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.w since the sparse map is not accounted for in hdr.Size. - if _, err := tw.w.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - return nil -} - -type ( - stringFormatter func([]byte, string) - numberFormatter func([]byte, int64) -) - -// templateV7Plus fills out the V7 fields of a block using values from hdr. -// It also fills out fields (uname, gname, devmajor, devminor) that are -// shared in the USTAR, PAX, and GNU formats using the provided formatters. -// -// The block returned is only valid until the next call to -// templateV7Plus or writeRawFile. -func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block { - tw.blk.Reset() - - modTime := hdr.ModTime - if modTime.IsZero() { - modTime = time.Unix(0, 0) - } - - v7 := tw.blk.V7() - v7.TypeFlag()[0] = hdr.Typeflag - fmtStr(v7.Name(), hdr.Name) - fmtStr(v7.LinkName(), hdr.Linkname) - fmtNum(v7.Mode(), hdr.Mode) - fmtNum(v7.UID(), int64(hdr.Uid)) - fmtNum(v7.GID(), int64(hdr.Gid)) - fmtNum(v7.Size(), hdr.Size) - fmtNum(v7.ModTime(), modTime.Unix()) - - ustar := tw.blk.USTAR() - fmtStr(ustar.UserName(), hdr.Uname) - fmtStr(ustar.GroupName(), hdr.Gname) - fmtNum(ustar.DevMajor(), hdr.Devmajor) - fmtNum(ustar.DevMinor(), hdr.Devminor) - - return &tw.blk -} - -// writeRawFile writes a minimal file with the given name and flag type. -// It uses format to encode the header format and will write data as the body. -// It uses default values for all of the other fields (as BSD and GNU tar does). -func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error { - tw.blk.Reset() - - // Best effort for the filename. - name = toASCII(name) - if len(name) > nameSize { - name = name[:nameSize] - } - name = strings.TrimRight(name, "/") - - var f formatter - v7 := tw.blk.V7() - v7.TypeFlag()[0] = flag - f.formatString(v7.Name(), name) - f.formatOctal(v7.Mode(), 0) - f.formatOctal(v7.UID(), 0) - f.formatOctal(v7.GID(), 0) - f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB - f.formatOctal(v7.ModTime(), 0) - tw.blk.SetFormat(format) - if f.err != nil { - return f.err // Only occurs if size condition is violated - } - - // Write the header and data. - if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil { - return err - } - _, err := io.WriteString(tw, data) - return err -} - -// writeRawHeader writes the value of blk, regardless of its value. -// It sets up the Writer such that it can accept a file of the given size. -// If the flag is a special header-only flag, then the size is treated as zero. -func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error { - if err := tw.Flush(); err != nil { - return err - } - if _, err := tw.w.Write(blk[:]); err != nil { - return err - } - if isHeaderOnlyType(flag) { - size = 0 - } - tw.curr = ®FileWriter{tw.w, size} - tw.pad = blockPadding(size) - return nil -} - -// splitUSTARPath splits a path according to USTAR prefix and suffix rules. -// If the path is not splittable, then it will return ("", "", false). -func splitUSTARPath(name string) (prefix, suffix string, ok bool) { - length := len(name) - if length <= nameSize || !isASCII(name) { - return "", "", false - } else if length > prefixSize+1 { - length = prefixSize + 1 - } else if name[length-1] == '/' { - length-- - } - - i := strings.LastIndex(name[:length], "/") - nlen := len(name) - i - 1 // nlen is length of suffix - plen := i // plen is length of prefix - if i <= 0 || nlen > nameSize || nlen == 0 || plen > prefixSize { - return "", "", false - } - return name[:i], name[i+1:], true -} - -// Write writes to the current file in the tar archive. -// Write returns the error ErrWriteTooLong if more than -// Header.Size bytes are written after WriteHeader. -// -// Calling Write on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless -// of what the Header.Size claims. -func (tw *Writer) Write(b []byte) (int, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.Write(b) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// readFrom populates the content of the current file by reading from r. -// The bytes read must match the number of remaining bytes in the current file. -// -// If the current file is sparse and r is an io.ReadSeeker, -// then readFrom uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are all NULs. -// This always reads the last byte to ensure r is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tw *Writer) readFrom(r io.Reader) (int64, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.ReadFrom(r) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// Close closes the tar archive by flushing the padding, and writing the footer. -// If the current file (from a prior call to WriteHeader) is not fully written, -// then this returns an error. -func (tw *Writer) Close() error { - if tw.err == ErrWriteAfterClose { - return nil - } - if tw.err != nil { - return tw.err - } - - // Trailer: two zero blocks. - err := tw.Flush() - for i := 0; i < 2 && err == nil; i++ { - _, err = tw.w.Write(zeroBlock[:]) - } - - // Ensure all future actions are invalid. - tw.err = ErrWriteAfterClose - return err // Report IO errors -} - -// regFileWriter is a fileWriter for writing data to a regular file entry. -type regFileWriter struct { - w io.Writer // Underlying Writer - nb int64 // Number of remaining bytes to write -} - -func (fw *regFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > fw.nb - if overwrite { - b = b[:fw.nb] - } - if len(b) > 0 { - n, err = fw.w.Write(b) - fw.nb -= int64(n) - } - switch { - case err != nil: - return n, err - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) { - return io.Copy(struct{ io.Writer }{fw}, r) -} - -func (fw regFileWriter) LogicalRemaining() int64 { - return fw.nb -} -func (fw regFileWriter) PhysicalRemaining() int64 { - return fw.nb -} - -// sparseFileWriter is a fileWriter for writing data to a sparse file entry. -type sparseFileWriter struct { - fw fileWriter // Underlying fileWriter - sp sparseDatas // Normalized list of data fragments - pos int64 // Current position in sparse file -} - -func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > sw.LogicalRemaining() - if overwrite { - b = b[:sw.LogicalRemaining()] - } - - b0 := b - endPos := sw.pos + int64(len(b)) - for endPos > sw.pos && err == nil { - var nf int // Bytes written in fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - bf := b[:min(int64(len(b)), dataStart-sw.pos)] - nf, err = zeroWriter{}.Write(bf) - } else { // In a data fragment - bf := b[:min(int64(len(b)), dataEnd-sw.pos)] - nf, err = sw.fw.Write(bf) - } - b = b[nf:] - sw.pos += int64(nf) - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) { - rs, ok := r.(io.ReadSeeker) - if ok { - if _, err := rs.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(struct{ io.Writer }{sw}, r) - } - - var readLastByte bool - pos0 := sw.pos - for sw.LogicalRemaining() > 0 && !readLastByte && err == nil { - var nf int64 // Size of fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - nf = dataStart - sw.pos - if sw.PhysicalRemaining() == 0 { - readLastByte = true - nf-- - } - _, err = rs.Seek(nf, io.SeekCurrent) - } else { // In a data fragment - nf = dataEnd - sw.pos - nf, err = io.CopyN(sw.fw, rs, nf) - } - sw.pos += nf - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // read a single byte to ensure the file is the right size. - if readLastByte && err == nil { - _, err = mustReadFull(rs, []byte{0}) - sw.pos++ - } - - n = sw.pos - pos0 - switch { - case err == io.EOF: - return n, io.ErrUnexpectedEOF - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - default: - return n, ensureEOF(rs) - } -} - -func (sw sparseFileWriter) LogicalRemaining() int64 { - return sw.sp[len(sw.sp)-1].endOffset() - sw.pos -} -func (sw sparseFileWriter) PhysicalRemaining() int64 { - return sw.fw.PhysicalRemaining() -} - -// zeroWriter may only be written with NULs, otherwise it returns errWriteHole. -type zeroWriter struct{} - -func (zeroWriter) Write(b []byte) (int, error) { - for i, c := range b { - if c != 0 { - return i, errWriteHole - } - } - return len(b), nil -} - -// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so. -func ensureEOF(r io.Reader) error { - n, err := tryReadFull(r, []byte{0}) - switch { - case n > 0: - return ErrWriteTooLong - case err == io.EOF: - return nil - default: - return err - } -} diff --git a/vendor/golang.org/x/sys/plan9/syscall_plan9.go b/vendor/golang.org/x/sys/plan9/syscall_plan9.go index d079d8116..761912237 100644 --- a/vendor/golang.org/x/sys/plan9/syscall_plan9.go +++ b/vendor/golang.org/x/sys/plan9/syscall_plan9.go @@ -19,13 +19,7 @@ import ( // A Note is a string describing a process note. // It implements the os.Signal interface. -type Note string - -func (n Note) Signal() {} - -func (n Note) String() string { - return string(n) -} +type Note = syscall.Note var ( Stdin = 0 diff --git a/vendor/golang.org/x/sys/unix/affinity_linux.go b/vendor/golang.org/x/sys/unix/affinity_linux.go index 3ea470387..acd6257fa 100644 --- a/vendor/golang.org/x/sys/unix/affinity_linux.go +++ b/vendor/golang.org/x/sys/unix/affinity_linux.go @@ -13,11 +13,19 @@ import ( const cpuSetSize = _CPU_SETSIZE / _NCPUBITS -// CPUSet represents a CPU affinity mask. +// CPUSet represents a bit mask of CPUs, to be used with [SchedGetaffinity], [SchedSetaffinity], +// and [SetMemPolicy]. +// +// Note this type can only represent CPU IDs 0 through 1023. +// Use [CPUSetDynamic]/[NewCPUSet] instead to avoid this limit. type CPUSet [cpuSetSize]cpuMask -func schedAffinity(trap uintptr, pid int, set *CPUSet) error { - _, _, e := RawSyscall(trap, uintptr(pid), uintptr(unsafe.Sizeof(*set)), uintptr(unsafe.Pointer(set))) +// CPUSetDynamic represents a bit mask of CPUs, to be used with [SchedGetaffinityDynamic], +// [SchedSetaffinityDynamic], and [SetMemPolicyDynamic]. Use [NewCPUSet] to allocate. +type CPUSetDynamic []cpuMask + +func schedAffinity(trap uintptr, pid int, size uintptr, ptr unsafe.Pointer) error { + _, _, e := RawSyscall(trap, uintptr(pid), uintptr(size), uintptr(ptr)) if e != 0 { return errnoErr(e) } @@ -27,13 +35,13 @@ func schedAffinity(trap uintptr, pid int, set *CPUSet) error { // SchedGetaffinity gets the CPU affinity mask of the thread specified by pid. // If pid is 0 the calling thread is used. func SchedGetaffinity(pid int, set *CPUSet) error { - return schedAffinity(SYS_SCHED_GETAFFINITY, pid, set) + return schedAffinity(SYS_SCHED_GETAFFINITY, pid, unsafe.Sizeof(*set), unsafe.Pointer(set)) } // SchedSetaffinity sets the CPU affinity mask of the thread specified by pid. // If pid is 0 the calling thread is used. func SchedSetaffinity(pid int, set *CPUSet) error { - return schedAffinity(SYS_SCHED_SETAFFINITY, pid, set) + return schedAffinity(SYS_SCHED_SETAFFINITY, pid, unsafe.Sizeof(*set), unsafe.Pointer(set)) } // Zero clears the set s, so that it contains no CPUs. @@ -45,9 +53,7 @@ func (s *CPUSet) Zero() { // will silently ignore any invalid CPU bits in [CPUSet] so this is an // efficient way of resetting the CPU affinity of a process. func (s *CPUSet) Fill() { - for i := range s { - s[i] = ^cpuMask(0) - } + cpuMaskFill(s[:]) } func cpuBitsIndex(cpu int) int { @@ -58,24 +64,27 @@ func cpuBitsMask(cpu int) cpuMask { return cpuMask(1 << (uint(cpu) % _NCPUBITS)) } -// Set adds cpu to the set s. -func (s *CPUSet) Set(cpu int) { +func cpuMaskFill(s []cpuMask) { + for i := range s { + s[i] = ^cpuMask(0) + } +} + +func cpuMaskSet(s []cpuMask, cpu int) { i := cpuBitsIndex(cpu) if i < len(s) { s[i] |= cpuBitsMask(cpu) } } -// Clear removes cpu from the set s. -func (s *CPUSet) Clear(cpu int) { +func cpuMaskClear(s []cpuMask, cpu int) { i := cpuBitsIndex(cpu) if i < len(s) { s[i] &^= cpuBitsMask(cpu) } } -// IsSet reports whether cpu is in the set s. -func (s *CPUSet) IsSet(cpu int) bool { +func cpuMaskIsSet(s []cpuMask, cpu int) bool { i := cpuBitsIndex(cpu) if i < len(s) { return s[i]&cpuBitsMask(cpu) != 0 @@ -83,11 +92,98 @@ func (s *CPUSet) IsSet(cpu int) bool { return false } -// Count returns the number of CPUs in the set s. -func (s *CPUSet) Count() int { +func cpuMaskCount(s []cpuMask) int { c := 0 for _, b := range s { c += bits.OnesCount64(uint64(b)) } return c } + +// Set adds cpu to the set s. If cpu is out of bounds for s, no action is taken. +func (s *CPUSet) Set(cpu int) { + cpuMaskSet(s[:], cpu) +} + +// Clear removes cpu from the set s. If cpu is out of bounds for s, no action is taken. +func (s *CPUSet) Clear(cpu int) { + cpuMaskClear(s[:], cpu) +} + +// IsSet reports whether cpu is in the set s. +func (s *CPUSet) IsSet(cpu int) bool { + return cpuMaskIsSet(s[:], cpu) +} + +// Count returns the number of CPUs in the set s. +func (s *CPUSet) Count() int { + return cpuMaskCount(s[:]) +} + +// NewCPUSet creates a CPU affinity mask capable of representing CPU IDs +// up to maxCPU (exclusive). +func NewCPUSet(maxCPU int) CPUSetDynamic { + numMasks := (maxCPU + _NCPUBITS - 1) / _NCPUBITS + if numMasks == 0 { + numMasks = 1 + } + return make(CPUSetDynamic, numMasks) +} + +// Zero clears the set s, so that it contains no CPUs. +func (s CPUSetDynamic) Zero() { + clear(s) +} + +// Fill adds all possible CPU bits to the set s. On Linux, [SchedSetaffinityDynamic] +// will silently ignore any invalid CPU bits in [CPUSetDynamic] so this is an +// efficient way of resetting the CPU affinity of a process. +func (s CPUSetDynamic) Fill() { + cpuMaskFill(s) +} + +// Set adds cpu to the set s. If cpu is out of bounds for s, no action is taken. +func (s CPUSetDynamic) Set(cpu int) { + cpuMaskSet(s, cpu) +} + +// Clear removes cpu from the set s. If cpu is out of bounds for s, no action is taken. +func (s CPUSetDynamic) Clear(cpu int) { + cpuMaskClear(s, cpu) +} + +// IsSet reports whether cpu is in the set s. +func (s CPUSetDynamic) IsSet(cpu int) bool { + return cpuMaskIsSet(s, cpu) +} + +// Count returns the number of CPUs in the set s. +func (s CPUSetDynamic) Count() int { + return cpuMaskCount(s) +} + +func (s CPUSetDynamic) size() uintptr { + return uintptr(len(s)) * unsafe.Sizeof(cpuMask(0)) +} + +func (s CPUSetDynamic) pointer() unsafe.Pointer { + if len(s) == 0 { + return nil + } + return unsafe.Pointer(&s[0]) +} + +// SchedGetaffinityDynamic gets the CPU affinity mask of the thread specified by pid. +// If pid is 0 the calling thread is used. +// +// If the set is smaller than the size of the affinity mask used by the kernel, +// [EINVAL] is returned. +func SchedGetaffinityDynamic(pid int, set CPUSetDynamic) error { + return schedAffinity(SYS_SCHED_GETAFFINITY, pid, set.size(), set.pointer()) +} + +// SchedSetaffinityDynamic sets the CPU affinity mask of the thread specified by pid. +// If pid is 0 the calling thread is used. +func SchedSetaffinityDynamic(pid int, set CPUSetDynamic) error { + return schedAffinity(SYS_SCHED_SETAFFINITY, pid, set.size(), set.pointer()) +} diff --git a/vendor/golang.org/x/sys/unix/mkall.sh b/vendor/golang.org/x/sys/unix/mkall.sh index d0ed61191..f6ddee1ae 100644 --- a/vendor/golang.org/x/sys/unix/mkall.sh +++ b/vendor/golang.org/x/sys/unix/mkall.sh @@ -51,7 +51,7 @@ if [[ "$GOOS" = "linux" ]]; then # Files generated through docker (use $cmd so you can Ctl-C the build or run) set -e $cmd docker build --tag generate:$GOOS $GOOS - $cmd docker run --interactive --tty --volume $(cd -- "$(dirname -- "$0")/.." && pwd):/build generate:$GOOS + $cmd docker run --rm --interactive --tty --volume $(cd -- "$(dirname -- "$0")/.." && pwd):/build generate:$GOOS exit fi diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index 06c0eea6f..f7b82bcca 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -2644,8 +2644,12 @@ func SchedGetAttr(pid int, flags uint) (*SchedAttr, error) { //sys Cachestat(fd uint, crange *CachestatRange, cstat *Cachestat_t, flags uint) (err error) //sys Mseal(b []byte, flags uint) (err error) -//sys setMemPolicy(mode int, mask *CPUSet, size int) (err error) = SYS_SET_MEMPOLICY +//sys setMemPolicy(mode int, mask unsafe.Pointer, size uintptr) (err error) = SYS_SET_MEMPOLICY func SetMemPolicy(mode int, mask *CPUSet) error { - return setMemPolicy(mode, mask, _CPU_SETSIZE) + return setMemPolicy(mode, unsafe.Pointer(mask), _CPU_SETSIZE) +} + +func SetMemPolicyDynamic(mode int, mask CPUSetDynamic) error { + return setMemPolicy(mode, mask.pointer(), mask.size()) } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm.go index cd2dd797f..ecf92bfa2 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm.go @@ -82,6 +82,9 @@ func Time(t *Time_t) (Time_t, error) { } func Utime(path string, buf *Utimbuf) error { + if buf == nil { + return Utimes(path, nil) + } tv := []Timeval{ {Sec: buf.Actime}, {Sec: buf.Modtime}, diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go index 745e5c7e6..173738077 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go @@ -113,6 +113,9 @@ func Time(t *Time_t) (Time_t, error) { } func Utime(path string, buf *Utimbuf) error { + if buf == nil { + return Utimes(path, nil) + } tv := []Timeval{ {Sec: buf.Actime}, {Sec: buf.Modtime}, diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go index dd2262a40..a3fd1d0b8 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go @@ -150,6 +150,9 @@ func Time(t *Time_t) (Time_t, error) { } func Utime(path string, buf *Utimbuf) error { + if buf == nil { + return Utimes(path, nil) + } tv := []Timeval{ {Sec: buf.Actime}, {Sec: buf.Modtime}, diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go index 8cf3670bd..fc5543c5f 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go @@ -112,6 +112,9 @@ func Time(t *Time_t) (Time_t, error) { } func Utime(path string, buf *Utimbuf) error { + if buf == nil { + return Utimes(path, nil) + } tv := []Timeval{ {Sec: buf.Actime}, {Sec: buf.Modtime}, diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index 8935d10a3..886f5de5b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -2241,8 +2241,8 @@ func Mseal(b []byte, flags uint) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setMemPolicy(mode int, mask *CPUSet, size int) (err error) { - _, _, e1 := Syscall(SYS_SET_MEMPOLICY, uintptr(mode), uintptr(unsafe.Pointer(mask)), uintptr(size)) +func setMemPolicy(mode int, mask unsafe.Pointer, size uintptr) (err error) { + _, _, e1 := Syscall(SYS_SET_MEMPOLICY, uintptr(mode), uintptr(mask), uintptr(size)) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index c1a467017..45476a73c 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -593,110 +593,115 @@ const ( ) const ( - NDA_UNSPEC = 0x0 - NDA_DST = 0x1 - NDA_LLADDR = 0x2 - NDA_CACHEINFO = 0x3 - NDA_PROBES = 0x4 - NDA_VLAN = 0x5 - NDA_PORT = 0x6 - NDA_VNI = 0x7 - NDA_IFINDEX = 0x8 - NDA_MASTER = 0x9 - NDA_LINK_NETNSID = 0xa - NDA_SRC_VNI = 0xb - NTF_USE = 0x1 - NTF_SELF = 0x2 - NTF_MASTER = 0x4 - NTF_PROXY = 0x8 - NTF_EXT_LEARNED = 0x10 - NTF_OFFLOADED = 0x20 - NTF_ROUTER = 0x80 - NUD_INCOMPLETE = 0x1 - NUD_REACHABLE = 0x2 - NUD_STALE = 0x4 - NUD_DELAY = 0x8 - NUD_PROBE = 0x10 - NUD_FAILED = 0x20 - NUD_NOARP = 0x40 - NUD_PERMANENT = 0x80 - NUD_NONE = 0x0 - IFA_UNSPEC = 0x0 - IFA_ADDRESS = 0x1 - IFA_LOCAL = 0x2 - IFA_LABEL = 0x3 - IFA_BROADCAST = 0x4 - IFA_ANYCAST = 0x5 - IFA_CACHEINFO = 0x6 - IFA_MULTICAST = 0x7 - IFA_FLAGS = 0x8 - IFA_RT_PRIORITY = 0x9 - IFA_TARGET_NETNSID = 0xa - IFAL_LABEL = 0x2 - IFAL_ADDRESS = 0x1 - RT_SCOPE_UNIVERSE = 0x0 - RT_SCOPE_SITE = 0xc8 - RT_SCOPE_LINK = 0xfd - RT_SCOPE_HOST = 0xfe - RT_SCOPE_NOWHERE = 0xff - RT_TABLE_UNSPEC = 0x0 - RT_TABLE_COMPAT = 0xfc - RT_TABLE_DEFAULT = 0xfd - RT_TABLE_MAIN = 0xfe - RT_TABLE_LOCAL = 0xff - RT_TABLE_MAX = 0xffffffff - RTA_UNSPEC = 0x0 - RTA_DST = 0x1 - RTA_SRC = 0x2 - RTA_IIF = 0x3 - RTA_OIF = 0x4 - RTA_GATEWAY = 0x5 - RTA_PRIORITY = 0x6 - RTA_PREFSRC = 0x7 - RTA_METRICS = 0x8 - RTA_MULTIPATH = 0x9 - RTA_FLOW = 0xb - RTA_CACHEINFO = 0xc - RTA_TABLE = 0xf - RTA_MARK = 0x10 - RTA_MFC_STATS = 0x11 - RTA_VIA = 0x12 - RTA_NEWDST = 0x13 - RTA_PREF = 0x14 - RTA_ENCAP_TYPE = 0x15 - RTA_ENCAP = 0x16 - RTA_EXPIRES = 0x17 - RTA_PAD = 0x18 - RTA_UID = 0x19 - RTA_TTL_PROPAGATE = 0x1a - RTA_IP_PROTO = 0x1b - RTA_SPORT = 0x1c - RTA_DPORT = 0x1d - RTN_UNSPEC = 0x0 - RTN_UNICAST = 0x1 - RTN_LOCAL = 0x2 - RTN_BROADCAST = 0x3 - RTN_ANYCAST = 0x4 - RTN_MULTICAST = 0x5 - RTN_BLACKHOLE = 0x6 - RTN_UNREACHABLE = 0x7 - RTN_PROHIBIT = 0x8 - RTN_THROW = 0x9 - RTN_NAT = 0xa - RTN_XRESOLVE = 0xb - SizeofNlMsghdr = 0x10 - SizeofNlMsgerr = 0x14 - SizeofRtGenmsg = 0x1 - SizeofNlAttr = 0x4 - SizeofRtAttr = 0x4 - SizeofIfInfomsg = 0x10 - SizeofIfAddrmsg = 0x8 - SizeofIfAddrlblmsg = 0xc - SizeofIfaCacheinfo = 0x10 - SizeofRtMsg = 0xc - SizeofRtNexthop = 0x8 - SizeofNdUseroptmsg = 0x10 - SizeofNdMsg = 0xc + NDA_UNSPEC = 0x0 + NDA_DST = 0x1 + NDA_LLADDR = 0x2 + NDA_CACHEINFO = 0x3 + NDA_PROBES = 0x4 + NDA_VLAN = 0x5 + NDA_PORT = 0x6 + NDA_VNI = 0x7 + NDA_IFINDEX = 0x8 + NDA_MASTER = 0x9 + NDA_LINK_NETNSID = 0xa + NDA_SRC_VNI = 0xb + NTF_USE = 0x1 + NTF_SELF = 0x2 + NTF_MASTER = 0x4 + NTF_PROXY = 0x8 + NTF_EXT_LEARNED = 0x10 + NTF_OFFLOADED = 0x20 + NTF_ROUTER = 0x80 + NUD_INCOMPLETE = 0x1 + NUD_REACHABLE = 0x2 + NUD_STALE = 0x4 + NUD_DELAY = 0x8 + NUD_PROBE = 0x10 + NUD_FAILED = 0x20 + NUD_NOARP = 0x40 + NUD_PERMANENT = 0x80 + NUD_NONE = 0x0 + IFA_UNSPEC = 0x0 + IFA_ADDRESS = 0x1 + IFA_LOCAL = 0x2 + IFA_LABEL = 0x3 + IFA_BROADCAST = 0x4 + IFA_ANYCAST = 0x5 + IFA_CACHEINFO = 0x6 + IFA_MULTICAST = 0x7 + IFA_FLAGS = 0x8 + IFA_RT_PRIORITY = 0x9 + IFA_TARGET_NETNSID = 0xa + IFAL_LABEL = 0x2 + IFAL_ADDRESS = 0x1 + RT_SCOPE_UNIVERSE = 0x0 + RT_SCOPE_SITE = 0xc8 + RT_SCOPE_LINK = 0xfd + RT_SCOPE_HOST = 0xfe + RT_SCOPE_NOWHERE = 0xff + RT_TABLE_UNSPEC = 0x0 + RT_TABLE_COMPAT = 0xfc + RT_TABLE_DEFAULT = 0xfd + RT_TABLE_MAIN = 0xfe + RT_TABLE_LOCAL = 0xff + RT_TABLE_MAX = 0xffffffff + RTA_UNSPEC = 0x0 + RTA_DST = 0x1 + RTA_SRC = 0x2 + RTA_IIF = 0x3 + RTA_OIF = 0x4 + RTA_GATEWAY = 0x5 + RTA_PRIORITY = 0x6 + RTA_PREFSRC = 0x7 + RTA_METRICS = 0x8 + RTA_MULTIPATH = 0x9 + RTA_FLOW = 0xb + RTA_CACHEINFO = 0xc + RTA_TABLE = 0xf + RTA_MARK = 0x10 + RTA_MFC_STATS = 0x11 + RTA_VIA = 0x12 + RTA_NEWDST = 0x13 + RTA_PREF = 0x14 + RTA_ENCAP_TYPE = 0x15 + RTA_ENCAP = 0x16 + RTA_EXPIRES = 0x17 + RTA_PAD = 0x18 + RTA_UID = 0x19 + RTA_TTL_PROPAGATE = 0x1a + RTA_IP_PROTO = 0x1b + RTA_SPORT = 0x1c + RTA_DPORT = 0x1d + RTN_UNSPEC = 0x0 + RTN_UNICAST = 0x1 + RTN_LOCAL = 0x2 + RTN_BROADCAST = 0x3 + RTN_ANYCAST = 0x4 + RTN_MULTICAST = 0x5 + RTN_BLACKHOLE = 0x6 + RTN_UNREACHABLE = 0x7 + RTN_PROHIBIT = 0x8 + RTN_THROW = 0x9 + RTN_NAT = 0xa + RTN_XRESOLVE = 0xb + PREFIX_UNSPEC = 0x0 + PREFIX_ADDRESS = 0x1 + PREFIX_CACHEINFO = 0x2 + SizeofNlMsghdr = 0x10 + SizeofNlMsgerr = 0x14 + SizeofRtGenmsg = 0x1 + SizeofNlAttr = 0x4 + SizeofRtAttr = 0x4 + SizeofIfInfomsg = 0x10 + SizeofPrefixmsg = 0xc + SizeofPrefixCacheinfo = 0x8 + SizeofIfAddrmsg = 0x8 + SizeofIfAddrlblmsg = 0xc + SizeofIfaCacheinfo = 0x10 + SizeofRtMsg = 0xc + SizeofRtNexthop = 0x8 + SizeofNdUseroptmsg = 0x10 + SizeofNdMsg = 0xc ) type NlMsghdr struct { @@ -735,6 +740,22 @@ type IfInfomsg struct { Change uint32 } +type Prefixmsg struct { + Family uint8 + Pad1 uint8 + Pad2 uint16 + Ifindex int32 + Type uint8 + Len uint8 + Flags uint8 + Pad3 uint8 +} + +type PrefixCacheinfo struct { + Preferred_time uint32 + Valid_time uint32 +} + type IfAddrmsg struct { Family uint8 Prefixlen uint8 diff --git a/vendor/golang.org/x/sys/windows/aliases.go b/vendor/golang.org/x/sys/windows/aliases.go index 16f90560a..96317966e 100644 --- a/vendor/golang.org/x/sys/windows/aliases.go +++ b/vendor/golang.org/x/sys/windows/aliases.go @@ -8,5 +8,6 @@ package windows import "syscall" +type Signal = syscall.Signal type Errno = syscall.Errno type SysProcAttr = syscall.SysProcAttr diff --git a/vendor/golang.org/x/sys/windows/dll_windows.go b/vendor/golang.org/x/sys/windows/dll_windows.go index 3ca814f54..1157b06d8 100644 --- a/vendor/golang.org/x/sys/windows/dll_windows.go +++ b/vendor/golang.org/x/sys/windows/dll_windows.go @@ -163,42 +163,7 @@ func (p *Proc) Addr() uintptr { // (according to the semantics of the specific function being called) before consulting // the error. The error will be guaranteed to contain windows.Errno. func (p *Proc) Call(a ...uintptr) (r1, r2 uintptr, lastErr error) { - switch len(a) { - case 0: - return syscall.Syscall(p.Addr(), uintptr(len(a)), 0, 0, 0) - case 1: - return syscall.Syscall(p.Addr(), uintptr(len(a)), a[0], 0, 0) - case 2: - return syscall.Syscall(p.Addr(), uintptr(len(a)), a[0], a[1], 0) - case 3: - return syscall.Syscall(p.Addr(), uintptr(len(a)), a[0], a[1], a[2]) - case 4: - return syscall.Syscall6(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], 0, 0) - case 5: - return syscall.Syscall6(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], 0) - case 6: - return syscall.Syscall6(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5]) - case 7: - return syscall.Syscall9(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], 0, 0) - case 8: - return syscall.Syscall9(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], 0) - case 9: - return syscall.Syscall9(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8]) - case 10: - return syscall.Syscall12(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], 0, 0) - case 11: - return syscall.Syscall12(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], 0) - case 12: - return syscall.Syscall12(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11]) - case 13: - return syscall.Syscall15(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], 0, 0) - case 14: - return syscall.Syscall15(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], 0) - case 15: - return syscall.Syscall15(p.Addr(), uintptr(len(a)), a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14]) - default: - panic("Call " + p.Name + " with too many arguments " + itoa(len(a)) + ".") - } + return syscall.SyscallN(p.Addr(), a...) } // A LazyDLL implements access to a single DLL. diff --git a/vendor/golang.org/x/sys/windows/security_windows.go b/vendor/golang.org/x/sys/windows/security_windows.go index a8b0364c7..6c955cea1 100644 --- a/vendor/golang.org/x/sys/windows/security_windows.go +++ b/vendor/golang.org/x/sys/windows/security_windows.go @@ -1438,13 +1438,17 @@ func GetSecurityInfo(handle Handle, objectType SE_OBJECT_TYPE, securityInformati } // GetNamedSecurityInfo queries the security information for a given named object and returns the self-relative security -// descriptor result on the Go heap. +// descriptor result on the Go heap. The security descriptor might be nil, even when err is nil, if the object exists +// but has no security descriptor. func GetNamedSecurityInfo(objectName string, objectType SE_OBJECT_TYPE, securityInformation SECURITY_INFORMATION) (sd *SECURITY_DESCRIPTOR, err error) { var winHeapSD *SECURITY_DESCRIPTOR err = getNamedSecurityInfo(objectName, objectType, securityInformation, nil, nil, nil, nil, &winHeapSD) if err != nil { return } + if winHeapSD == nil { + return nil, nil + } defer LocalFree(Handle(unsafe.Pointer(winHeapSD))) return winHeapSD.copySelfRelativeSecurityDescriptor(), nil } diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 738a9f212..453a7b97f 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -892,9 +892,13 @@ const socket_error = uintptr(^uint32(0)) //sys MultiByteToWideChar(codePage uint32, dwFlags uint32, str *byte, nstr int32, wchar *uint16, nwchar int32) (nwrite int32, err error) = kernel32.MultiByteToWideChar //sys getBestInterfaceEx(sockaddr unsafe.Pointer, pdwBestIfIndex *uint32) (errcode error) = iphlpapi.GetBestInterfaceEx //sys GetIfEntry2Ex(level uint32, row *MibIfRow2) (errcode error) = iphlpapi.GetIfEntry2Ex +//sys GetIfTable2Ex(level uint32, table **MibIfTable2) (errcode error) = iphlpapi.GetIfTable2Ex //sys GetIpForwardEntry2(row *MibIpForwardRow2) (errcode error) = iphlpapi.GetIpForwardEntry2 //sys GetIpForwardTable2(family uint16, table **MibIpForwardTable2) (errcode error) = iphlpapi.GetIpForwardTable2 +//sys GetIpInterfaceEntry(row *MibIpInterfaceRow) (errcode error) = iphlpapi.GetIpInterfaceEntry +//sys GetIpInterfaceTable(family uint16, table **MibIpInterfaceTable) (errcode error) = iphlpapi.GetIpInterfaceTable //sys GetUnicastIpAddressEntry(row *MibUnicastIpAddressRow) (errcode error) = iphlpapi.GetUnicastIpAddressEntry +//sys GetUnicastIpAddressTable(family uint16, table **MibUnicastIpAddressTable) (errcode error) = iphlpapi.GetUnicastIpAddressTable //sys FreeMibTable(memory unsafe.Pointer) = iphlpapi.FreeMibTable //sys NotifyIpInterfaceChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) = iphlpapi.NotifyIpInterfaceChange //sys NotifyRouteChange2(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) = iphlpapi.NotifyRouteChange2 @@ -1490,20 +1494,6 @@ func Getgid() (gid int) { return -1 } func Getegid() (egid int) { return -1 } func Getgroups() (gids []int, err error) { return nil, syscall.EWINDOWS } -type Signal int - -func (s Signal) Signal() {} - -func (s Signal) String() string { - if 0 <= s && int(s) < len(signals) { - str := signals[s] - if str != "" { - return str - } - } - return "signal " + itoa(int(s)) -} - func LoadCreateSymbolicLink() error { return procCreateSymbolicLinkW.Find() } @@ -1707,10 +1697,13 @@ func NewNTUnicodeString(s string) (*NTUnicodeString, error) { if err != nil { return nil, err } - n := uint16(len(s16) * 2) + n := len(s16) * 2 + if n > (1<<16)-1 { + return nil, syscall.EINVAL + } return &NTUnicodeString{ - Length: n - 2, // subtract 2 bytes for the NULL terminator - MaximumLength: n, + Length: uint16(n) - 2, // subtract 2 bytes for the NULL terminator + MaximumLength: uint16(n), Buffer: &s16[0], }, nil } diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index d5658a138..d82299e33 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -2320,6 +2320,21 @@ type MibIfRow2 struct { OutQLen uint64 } +// MIB_IF_TABLE_LEVEL enumeration from netioapi.h or +// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ne-netioapi-mib_if_table_level. +const ( + MibIfTableNormal = 0 + MibIfTableRaw = 1 + MibIfTableNormalWithoutStatistics = 2 +) + +// MibIfTable2 contains a table of logical and physical interface entries. See +// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_if_table2. +type MibIfTable2 struct { + NumEntries uint32 + Table [1]MibIfRow2 +} + // IP_ADDRESS_PREFIX stores an IP address prefix. See // https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-ip_address_prefix. type IpAddressPrefix struct { @@ -2413,6 +2428,13 @@ type MibUnicastIpAddressRow struct { CreationTimeStamp Filetime } +// MibUnicastIpAddressTable contains a table of unicast IP address entries. See +// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_unicastipaddress_table. +type MibUnicastIpAddressTable struct { + NumEntries uint32 + Table [1]MibUnicastIpAddressRow +} + const ScopeLevelCount = 16 // MIB_IPINTERFACE_ROW stores interface management information for a particular IP address family on a network interface. @@ -2455,6 +2477,13 @@ type MibIpInterfaceRow struct { DisableDefaultRoutes uint8 } +// MibIpInterfaceTable contains a table of IP interface entries. See +// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipinterface_table. +type MibIpInterfaceTable struct { + NumEntries uint32 + Table [1]MibIpInterfaceRow +} + // Console related constants used for the mode parameter to SetConsoleMode. See // https://docs.microsoft.com/en-us/windows/console/setconsolemode for details. diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index fe7a4ea12..a506ac0f1 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -188,9 +188,13 @@ var ( procGetBestInterfaceEx = modiphlpapi.NewProc("GetBestInterfaceEx") procGetIfEntry = modiphlpapi.NewProc("GetIfEntry") procGetIfEntry2Ex = modiphlpapi.NewProc("GetIfEntry2Ex") + procGetIfTable2Ex = modiphlpapi.NewProc("GetIfTable2Ex") procGetIpForwardEntry2 = modiphlpapi.NewProc("GetIpForwardEntry2") procGetIpForwardTable2 = modiphlpapi.NewProc("GetIpForwardTable2") + procGetIpInterfaceEntry = modiphlpapi.NewProc("GetIpInterfaceEntry") + procGetIpInterfaceTable = modiphlpapi.NewProc("GetIpInterfaceTable") procGetUnicastIpAddressEntry = modiphlpapi.NewProc("GetUnicastIpAddressEntry") + procGetUnicastIpAddressTable = modiphlpapi.NewProc("GetUnicastIpAddressTable") procNotifyIpInterfaceChange = modiphlpapi.NewProc("NotifyIpInterfaceChange") procNotifyRouteChange2 = modiphlpapi.NewProc("NotifyRouteChange2") procNotifyUnicastIpAddressChange = modiphlpapi.NewProc("NotifyUnicastIpAddressChange") @@ -1674,6 +1678,14 @@ func GetIfEntry2Ex(level uint32, row *MibIfRow2) (errcode error) { return } +func GetIfTable2Ex(level uint32, table **MibIfTable2) (errcode error) { + r0, _, _ := syscall.SyscallN(procGetIfTable2Ex.Addr(), uintptr(level), uintptr(unsafe.Pointer(table))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + func GetIpForwardEntry2(row *MibIpForwardRow2) (errcode error) { r0, _, _ := syscall.SyscallN(procGetIpForwardEntry2.Addr(), uintptr(unsafe.Pointer(row))) if r0 != 0 { @@ -1690,6 +1702,22 @@ func GetIpForwardTable2(family uint16, table **MibIpForwardTable2) (errcode erro return } +func GetIpInterfaceEntry(row *MibIpInterfaceRow) (errcode error) { + r0, _, _ := syscall.SyscallN(procGetIpInterfaceEntry.Addr(), uintptr(unsafe.Pointer(row))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + +func GetIpInterfaceTable(family uint16, table **MibIpInterfaceTable) (errcode error) { + r0, _, _ := syscall.SyscallN(procGetIpInterfaceTable.Addr(), uintptr(family), uintptr(unsafe.Pointer(table))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + func GetUnicastIpAddressEntry(row *MibUnicastIpAddressRow) (errcode error) { r0, _, _ := syscall.SyscallN(procGetUnicastIpAddressEntry.Addr(), uintptr(unsafe.Pointer(row))) if r0 != 0 { @@ -1698,6 +1726,14 @@ func GetUnicastIpAddressEntry(row *MibUnicastIpAddressRow) (errcode error) { return } +func GetUnicastIpAddressTable(family uint16, table **MibUnicastIpAddressTable) (errcode error) { + r0, _, _ := syscall.SyscallN(procGetUnicastIpAddressTable.Addr(), uintptr(family), uintptr(unsafe.Pointer(table))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + func NotifyIpInterfaceChange(family uint16, callback uintptr, callerContext unsafe.Pointer, initialNotification bool, notificationHandle *Handle) (errcode error) { var _p0 uint32 if initialNotification { diff --git a/vendor/modules.txt b/vendor/modules.txt index e0f4ca237..c376cefa5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -43,26 +43,19 @@ github.com/chai2010/gettext-go github.com/chai2010/gettext-go/mo github.com/chai2010/gettext-go/plural github.com/chai2010/gettext-go/po -# github.com/containerd/stargz-snapshotter/estargz v0.18.2 -## explicit; go 1.24.0 -github.com/containerd/stargz-snapshotter/estargz -github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/cpuguy83/go-md2man/v2 v2.0.7 ## explicit; go 1.12 github.com/cpuguy83/go-md2man/v2/md2man # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/docker/cli v29.2.1+incompatible +# github.com/docker/cli v29.4.3+incompatible ## explicit github.com/docker/cli/cli/config github.com/docker/cli/cli/config/configfile github.com/docker/cli/cli/config/credentials github.com/docker/cli/cli/config/memorystore github.com/docker/cli/cli/config/types -# github.com/docker/distribution v2.8.3+incompatible -## explicit -github.com/docker/distribution/registry/client/auth/challenge # github.com/docker/docker-credential-helpers v0.9.3 ## explicit; go 1.21 github.com/docker/docker-credential-helpers/client @@ -205,11 +198,10 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.21.2 -## explicit; go 1.25.6 +# github.com/google/go-containerregistry v0.21.6 +## explicit; go 1.25.0 github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/compression -github.com/google/go-containerregistry/internal/estargz github.com/google/go-containerregistry/internal/gzip github.com/google/go-containerregistry/internal/redact github.com/google/go-containerregistry/internal/retry @@ -226,6 +218,7 @@ github.com/google/go-containerregistry/pkg/v1/match github.com/google/go-containerregistry/pkg/v1/mutate github.com/google/go-containerregistry/pkg/v1/partial github.com/google/go-containerregistry/pkg/v1/remote +github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge github.com/google/go-containerregistry/pkg/v1/remote/transport github.com/google/go-containerregistry/pkg/v1/stream github.com/google/go-containerregistry/pkg/v1/tarball @@ -253,8 +246,8 @@ github.com/jbenet/go-context/io # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.18.4 -## explicit; go 1.23 +# github.com/klauspost/compress v1.18.6 +## explicit; go 1.24 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -269,9 +262,6 @@ github.com/liggitt/tabwriter # github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db ## explicit github.com/mitchellh/colorstring -# github.com/mitchellh/go-homedir v1.1.0 -## explicit -github.com/mitchellh/go-homedir # github.com/mitchellh/go-wordwrap v1.0.1 ## explicit; go 1.14 github.com/mitchellh/go-wordwrap @@ -380,8 +370,8 @@ github.com/shipwright-io/build/pkg/config github.com/shipwright-io/build/pkg/ctxlog github.com/shipwright-io/build/pkg/reconciler/buildrun/resources/sources github.com/shipwright-io/build/pkg/webhook -# github.com/sirupsen/logrus v1.9.3 -## explicit; go 1.13 +# github.com/sirupsen/logrus v1.9.4 +## explicit; go 1.17 github.com/sirupsen/logrus # github.com/spf13/cobra v1.10.2 ## explicit; go 1.15 @@ -410,9 +400,6 @@ github.com/tektoncd/pipeline/pkg/substitution # github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c ## explicit; go 1.13 github.com/texttheater/golang-levenshtein/levenshtein -# github.com/vbatts/tar-split v0.12.2 -## explicit; go 1.17 -github.com/vbatts/tar-split/archive/tar # github.com/x448/float16 v0.8.4 ## explicit; go 1.11 github.com/x448/float16 @@ -477,16 +464,16 @@ golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace golang.org/x/net/websocket -# golang.org/x/oauth2 v0.35.0 -## explicit; go 1.24.0 +# golang.org/x/oauth2 v0.36.0 +## explicit; go 1.25.0 golang.org/x/oauth2 golang.org/x/oauth2/internal -# golang.org/x/sync v0.19.0 -## explicit; go 1.24.0 +# golang.org/x/sync v0.20.0 +## explicit; go 1.25.0 golang.org/x/sync/errgroup golang.org/x/sync/semaphore -# golang.org/x/sys v0.41.0 -## explicit; go 1.24.0 +# golang.org/x/sys v0.44.0 +## explicit; go 1.25.0 golang.org/x/sys/plan9 golang.org/x/sys/unix golang.org/x/sys/windows @@ -652,6 +639,8 @@ gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.4.0 ## explicit; go 1.15 gopkg.in/yaml.v2 +# gotest.tools/v3 v3.5.2 +## explicit; go 1.17 # k8s.io/api v0.34.4 ## explicit; go 1.24.0 k8s.io/api/admission/v1