diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 019ea84..dae23ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.25' - name: Unit tests run: go test ./... diff --git a/.github/workflows/comment-sandbox.yaml b/.github/workflows/comment-sandbox.yaml index 3e067eb..cbbe148 100644 --- a/.github/workflows/comment-sandbox.yaml +++ b/.github/workflows/comment-sandbox.yaml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24" + go-version: "1.25" - name: Build pipekit run: make build diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1431a07..e8722b7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,7 +23,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.25' cache: true - name: Run tests diff --git a/README.md b/README.md index 67eb61d..498b437 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@
Pipekit — CI/CD pipeline toolkit

- Go Version + Go Version OS Support License

@@ -52,6 +52,12 @@ pipekit wait url http://localhost:8080/healthz --timeout 150s pipekit wait grpc localhost:50051 --service my.package.Worker --timeout 60s pipekit wait ws ws://localhost:8080/events --timeout 60s +# Request an API and extract JSON without curl+jq +pipekit http get https://api.example.com/release --expect-status 200 --jq .tag --raw + +# Pack cross-platform release archives +pipekit archive pack dist/app.tar.zst ./bin/app README.md + # Retry a flaky command with exponential backoff pipekit retry run --attempts 5 --delay 5s --backoff -- helm upgrade --install myapp ./chart @@ -81,6 +87,7 @@ More end-to-end recipes → **[docs/EXAMPLES.md](docs/EXAMPLES.md)** | `cache-key` | Deterministic SHA256 cache keys from files / globs / composite parts | [↗](docs/COMMANDS.md#cache-key) | | `checksum` | Generate / verify release checksums for artifact files | [↗](docs/COMMANDS.md#checksum) | | `artifact` | Assert artifacts exist and generate size/SHA256 manifests | [↗](docs/COMMANDS.md#artifact) | +| `archive` | Pack, list, and unpack zip/tar/tar.gz/tar.xz/tar.zst archives | [↗](docs/COMMANDS.md#archive) | | `git` | CI-friendly git metadata: ref, SHA, tags, dirty state | [↗](docs/COMMANDS.md#git) | | `changelog` | Generate release notes from git commit ranges | [↗](docs/COMMANDS.md#changelog) | | `config` | Resolve env-specific config maps; map branches to environments | [↗](docs/COMMANDS.md#config) | @@ -89,6 +96,7 @@ More end-to-end recipes → **[docs/EXAMPLES.md](docs/EXAMPLES.md)** | `json` / `yaml` | Get / set / del / deep-merge / convert / pretty / table on JSON, YAML, TOML, CSV | [↗](docs/COMMANDS.md#json) | | `render` | Render Go templates with a sprig-like FuncMap and stacked `--values` files | [↗](docs/COMMANDS.md#render) | | `exec` | Unified retry + mask + tee + timeout command runner | [↗](docs/COMMANDS.md#exec) | +| `http` | Curl-like HTTP requests, JSON extraction, uploads, and chained flows | [↗](docs/COMMANDS.md#http) | | `url parse` | Split a URL into `SCHEME / HOST / PORT / USER / PASSWORD / PATH / QUERY` env vars | [↗](docs/COMMANDS.md#url) | | `image parse` | Split a container image ref into registry / repository / tag / digest | [↗](docs/COMMANDS.md#image) | | `time` | RFC3339 / unix / tag / compact / iso timestamps; format conversion; arithmetic | [↗](docs/COMMANDS.md#time) | diff --git a/actions/archive.go b/actions/archive.go new file mode 100644 index 0000000..89be0a2 --- /dev/null +++ b/actions/archive.go @@ -0,0 +1,92 @@ +package actions + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/AxeForging/pipekit/services" + + "github.com/urfave/cli" +) + +// ArchiveCommand returns archive pack/list/unpack helpers. +func ArchiveCommand() cli.Command { + return cli.Command{ + Name: "archive", + Usage: "pack, list, and unpack tar/zip archives", + Subcommands: []cli.Command{ + { + Name: "pack", + Usage: "create an archive from files or directories", + ArgsUsage: "OUTPUT INPUT...", + Flags: []cli.Flag{ + cli.StringFlag{Name: "format, f", Usage: "zip, tar, tar.gz, tar.xz, or tar.zst (default: detect from output)"}, + }, + Action: func(c *cli.Context) error { + args := c.Args() + if len(args) < 2 { + return cli.NewExitError("usage: archive pack OUTPUT INPUT...", 1) + } + if err := services.PackArchive(args[0], args[1:], c.String("format")); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil + }, + }, + { + Name: "unpack", + Usage: "extract an archive", + ArgsUsage: "ARCHIVE", + Flags: []cli.Flag{ + cli.StringFlag{Name: "dest, C", Value: ".", Usage: "destination directory"}, + cli.StringFlag{Name: "format, f", Usage: "zip, tar, tar.gz, tar.xz, or tar.zst (default: detect from input)"}, + cli.IntFlag{Name: "strip-components", Usage: "strip leading path components while extracting"}, + }, + Action: func(c *cli.Context) error { + input, err := firstArgOrErr(c, "ARCHIVE") + if err != nil { + return err + } + if err := services.UnpackArchive(input, c.String("dest"), c.String("format"), c.Int("strip-components")); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil + }, + }, + { + Name: "list", + Usage: "list archive entries", + ArgsUsage: "ARCHIVE", + Flags: []cli.Flag{ + cli.StringFlag{Name: "format, f", Usage: "zip, tar, tar.gz, tar.xz, or tar.zst (default: detect from input)"}, + cli.BoolFlag{Name: "json", Usage: "output entry metadata as JSON"}, + }, + Action: func(c *cli.Context) error { + input, err := firstArgOrErr(c, "ARCHIVE") + if err != nil { + return err + } + entries, err := services.ListArchive(input, c.String("format")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + if c.Bool("json") { + b, err := json.MarshalIndent(entries, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + return nil + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name) + } + fmt.Println(strings.Join(names, "\n")) + return nil + }, + }, + }, + } +} diff --git a/actions/http.go b/actions/http.go new file mode 100644 index 0000000..7f75b7c --- /dev/null +++ b/actions/http.go @@ -0,0 +1,281 @@ +package actions + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/AxeForging/pipekit/services" + + "github.com/urfave/cli" +) + +// HTTPCommand returns curl-like HTTP helpers. +func HTTPCommand() cli.Command { + return cli.Command{ + Name: "http", + Usage: "curl-like HTTP requests with status assertions and JSON extraction", + Subcommands: []cli.Command{ + httpMethodCommand(http.MethodGet), + httpMethodCommand(http.MethodPost), + httpMethodCommand(http.MethodPut), + httpMethodCommand(http.MethodPatch), + httpMethodCommand(http.MethodDelete), + httpChainCommand(), + }, + } +} + +func httpMethodCommand(method string) cli.Command { + return cli.Command{ + Name: strings.ToLower(method), + Usage: method + " an HTTP(S) URL", + ArgsUsage: "URL", + Flags: []cli.Flag{ + cli.StringSliceFlag{Name: "header, H", Usage: "request header as 'Name: value' (repeatable)"}, + cli.StringFlag{Name: "data, d", Usage: "raw request body"}, + cli.StringFlag{Name: "data-file", Usage: "read raw request body from file"}, + cli.StringFlag{Name: "json, j", Usage: "JSON request body"}, + cli.StringFlag{Name: "json-file", Usage: "read JSON request body from file"}, + cli.StringSliceFlag{Name: "form, F", Usage: "application/x-www-form-urlencoded field as key=value (repeatable)"}, + cli.StringSliceFlag{Name: "file", Usage: "multipart file field as name=path (repeatable)"}, + cli.StringFlag{Name: "expect-status", Usage: "comma-separated acceptable HTTP status codes"}, + cli.StringFlag{Name: "jq", Usage: "jq-style response JSON path to print"}, + cli.BoolFlag{Name: "raw, r", Usage: "print extracted strings raw"}, + cli.StringFlag{Name: "output, o", Usage: "write response body or extracted value to file"}, + outputKeyFlag(), + cli.StringFlag{Name: "timeout", Value: "30s", Usage: "request timeout"}, + cli.IntFlag{Name: "retry", Value: 1, Usage: "number of attempts"}, + cli.StringFlag{Name: "retry-delay", Value: "1s", Usage: "delay between retry attempts"}, + cli.BoolFlag{Name: "backoff", Usage: "exponential backoff between retries"}, + cli.BoolFlag{Name: "insecure, k", Usage: "skip TLS certificate verification"}, + cli.BoolFlag{Name: "verbose, v", Usage: "print response status and headers to stderr"}, + }, + Action: func(c *cli.Context) error { + urlStr, err := firstArgOrErr(c, "URL") + if err != nil { + return err + } + timeout, err := time.ParseDuration(c.String("timeout")) + if err != nil { + return cli.NewExitError("invalid timeout: "+err.Error(), 1) + } + retryDelay, err := time.ParseDuration(c.String("retry-delay")) + if err != nil { + return cli.NewExitError("invalid retry-delay: "+err.Error(), 1) + } + headers, err := services.ParseHTTPHeaders(c.StringSlice("header")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + body, contentType, err := services.BuildHTTPBody(c.String("data"), c.String("data-file"), c.String("json"), c.String("json-file"), c.StringSlice("form")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + if len(c.StringSlice("file")) > 0 { + if body != nil { + return cli.NewExitError("use --file without --data, --data-file, --json, --json-file, or --form", 1) + } + body, contentType, err = services.BuildMultipartBody(c.StringSlice("file")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + } + if contentType != "" { + if _, exists := headers["Content-Type"]; !exists { + headers["Content-Type"] = contentType + } + } + + expected, err := parseHTTPStatusList(c.String("expect-status")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + res, err := services.ExecuteHTTPRequest(services.HTTPRequestOptions{ + Method: method, + URL: urlStr, + Headers: headers, + Body: body, + Timeout: timeout, + ExpectedStatus: expected, + RetryAttempts: c.Int("retry"), + RetryDelay: retryDelay, + Backoff: c.Bool("backoff"), + InsecureTLS: c.Bool("insecure"), + }) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + if c.Bool("verbose") { + printHTTPVerbose(res) + } + + out := res.Body + if path := c.String("jq"); path != "" { + val, err := services.ExtractHTTPJSON(res.Body, path) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + out = []byte(formatHTTPResult(val, c.Bool("raw"))) + } + if output := c.String("output"); output != "" { + return os.WriteFile(output, out, 0644) + } + if outputKey := c.String("to-github-output"); outputKey != "" { + return services.WriteToGitHubOutputValue(outputKey, strings.TrimRight(string(out), "\n")) + } + fmt.Print(string(out)) + return nil + }, + } +} + +func httpChainCommand() cli.Command { + return cli.Command{ + Name: "chain", + Usage: "run a sequence of dependent HTTP requests from a JSON/YAML plan", + ArgsUsage: "PLAN_FILE|-", + Flags: []cli.Flag{ + cli.StringSliceFlag{Name: "header, H", Usage: "request header as 'Name: value' applied to every step"}, + cli.StringFlag{Name: "from", Usage: "plan format override for stdin or extensionless files: json or yaml"}, + cli.StringFlag{Name: "expect-status", Usage: "default acceptable HTTP status codes for steps"}, + cli.StringFlag{Name: "timeout", Value: "30s", Usage: "default request timeout"}, + cli.IntFlag{Name: "retry", Value: 1, Usage: "number of attempts per step"}, + cli.StringFlag{Name: "retry-delay", Value: "1s", Usage: "delay between retry attempts"}, + cli.BoolFlag{Name: "backoff", Usage: "exponential backoff between retries"}, + cli.BoolFlag{Name: "insecure, k", Usage: "skip TLS certificate verification"}, + cli.BoolFlag{Name: "verbose, v", Usage: "print per-step status to stderr"}, + outputKeyFlag(), + }, + Action: func(c *cli.Context) error { + file, err := firstArgOrErr(c, "PLAN_FILE") + if err != nil { + return err + } + timeout, err := time.ParseDuration(c.String("timeout")) + if err != nil { + return cli.NewExitError("invalid timeout: "+err.Error(), 1) + } + retryDelay, err := time.ParseDuration(c.String("retry-delay")) + if err != nil { + return cli.NewExitError("invalid retry-delay: "+err.Error(), 1) + } + expected, err := parseHTTPStatusList(c.String("expect-status")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + headers, err := services.ParseHTTPHeaders(c.StringSlice("header")) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + data, format, err := readHTTPChainPlanInput(c, file) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + doc, err := services.Decode(data, format) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + plan, err := services.DecodeHTTPChainPlan(doc) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + result, err := services.ExecuteHTTPChain(plan, services.HTTPRequestOptions{ + Headers: headers, + Timeout: timeout, + ExpectedStatus: expected, + RetryAttempts: c.Int("retry"), + RetryDelay: retryDelay, + Backoff: c.Bool("backoff"), + InsecureTLS: c.Bool("insecure"), + }) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + if c.Bool("verbose") { + for _, step := range result.Steps { + fmt.Fprintf(os.Stderr, "%s: HTTP %d\n", step.Name, step.StatusCode) + } + } + b, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + if outputKey := c.String("to-github-output"); outputKey != "" { + return services.WriteToGitHubOutputValue(outputKey, string(b)) + } + fmt.Println(string(b)) + return nil + }, + } +} + +func readHTTPChainPlanInput(c *cli.Context, file string) ([]byte, services.DataFormat, error) { + format := services.FormatYAML + if from := c.String("from"); from != "" { + format = services.FormatString(from) + } else if file != "-" { + if detected := services.DetectFormat(file); detected != "" { + format = detected + } + } + if file == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, "", fmt.Errorf("reading stdin: %w", err) + } + if len(strings.TrimSpace(string(data))) == 0 { + return nil, "", fmt.Errorf("empty HTTP chain plan on stdin") + } + return data, format, nil + } + data, err := os.ReadFile(file) + if err != nil { + return nil, "", err + } + return data, format, nil +} + +func parseHTTPStatusList(value string) ([]int, error) { + if value == "" { + return nil, nil + } + var out []int + for _, part := range strings.Split(value, ",") { + code, err := strconv.Atoi(strings.TrimSpace(part)) + if err != nil || code < 100 || code > 999 { + return nil, fmt.Errorf("invalid status code: %s", part) + } + out = append(out, code) + } + return out, nil +} + +func formatHTTPResult(v interface{}, raw bool) string { + if raw { + if s, ok := v.(string); ok { + return s + } + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) + } + return string(b) +} + +func printHTTPVerbose(res *services.HTTPResult) { + fmt.Fprintf(os.Stderr, "HTTP %d\n", res.StatusCode) + for name, values := range res.Headers { + for _, value := range values { + fmt.Fprintf(os.Stderr, "%s: %s\n", name, value) + } + } +} diff --git a/docs/AI/README.md b/docs/AI/README.md index fb11b24..2dbda8f 100644 --- a/docs/AI/README.md +++ b/docs/AI/README.md @@ -7,7 +7,7 @@ This document provides structured context for AI assistants working with the pip | Item | Value | |------|-------| | **Purpose** | CLI tool replacing bash one-liners in CI/CD pipelines | -| **Language** | Go 1.24+ | +| **Language** | Go 1.25+ | | **CLI Framework** | `urfave/cli` v1 | | **Repository** | https://github.com/AxeForging/pipekit | | **License** | MIT | diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 3053f52..93ab9ad 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -18,6 +18,7 @@ Full reference for every pipekit command and flag. For end-to-end pipeline recip - [`cache-key`](#cache-key) — deterministic cache keys - [`checksum`](#checksum) — release checksums - [`artifact`](#artifact) — artifact manifests and assertions +- [`archive`](#archive) — archive pack/list/unpack - [`git`](#git) — CI-friendly git metadata - [`changelog`](#changelog) — release notes from git history - [`config`](#config) — environment configuration @@ -25,6 +26,7 @@ Full reference for every pipekit command and flag. For end-to-end pipeline recip - [`json` / `yaml`](#json) — read · query · mutate · merge · convert · pretty · table - [`render`](#render) — file templating with sprig-like funcs - [`exec`](#exec) — unified retry + mask + tee + timeout runner +- [`http`](#http) — curl-like HTTP requests and chains - [`url`](#url) — URL parsing - [`image`](#image) — container image ref parsing - [`time`](#time) — timestamps, formatting, arithmetic @@ -637,6 +639,41 @@ pipekit artifact manifest "dist/pipekit-*" --pretty --output dist/artifacts.json --- +## archive + +Create, inspect, and extract archives without depending on platform-specific `tar`, `zip`, `xz`, or `zstd` installs. + +
+Examples + +```sh +# Pack release files +pipekit archive pack dist/app.tar.zst ./bin/app README.md + +# Maximum compression release archive +pipekit archive pack dist/source.tar.xz ./src ./go.mod + +# Windows-friendly archive +pipekit archive pack dist/app.zip ./bin/app.exe README.md + +# Inspect and unpack +pipekit archive list dist/app.tar.zst +pipekit archive list dist/app.zip --json +pipekit archive unpack dist/app.tar.zst --dest ./tmp --strip-components 1 +``` + +| Subcommand | Description | +|---|---| +| `archive pack OUTPUT INPUT...` | Create an archive from files/directories | +| `archive list ARCHIVE` | List entries, optionally as JSON | +| `archive unpack ARCHIVE` | Extract entries safely to a destination | + +Supported formats: `zip`, `tar`, `tar.gz`, `tar.xz`, `tar.zst`. Formats are detected from filenames unless `--format` is set. + +
+ +--- + ## git Read git metadata in formats that are easy to pass between CI steps. @@ -1115,6 +1152,100 @@ pipekit exec --attempts 4 --delay 10s --retry-on-stderr "rate limit|429" \ --- +## http + +Run curl-like HTTP(S) requests with retries, status checks, JSON extraction, and chained captures. + +
+Examples + +```sh +# GET and extract JSON without jq +pipekit http get https://api.example.com/releases/latest \ + --expect-status 200 \ + --jq .tag_name \ + --raw \ + --to-github-output tag + +# POST JSON with retries +pipekit http post https://api.example.com/deploys \ + --header "Authorization: Bearer $TOKEN" \ + --json '{"ref":"main"}' \ + --expect-status 201 \ + --retry 3 \ + --backoff + +# Multipart upload +pipekit http post https://uploads.example.com/artifacts \ + --file artifact=dist/app.tar.zst \ + --expect-status 200 + +# Chain dependent requests with captures and interpolation +pipekit http chain flow.yaml --expect-status 200 --verbose + +# The same plan can be passed inline with a heredoc +pipekit http chain - --expect-status 200 <<'YAML' +steps: + - name: auth + method: POST + url: https://api.example.com/token + json: '{"client":"ci"}' + capture: + token: .access_token + - name: deploy + method: POST + url: https://api.example.com/deploys/{{token}} + headers: + Authorization: Bearer {{token}} + json: '{"ref":"main"}' + expectStatus: [201] +YAML +``` + +`http chain` runs each step in order. Pass a plan file, or use `-` to read a JSON/YAML plan from stdin. Values captured from a response body with jq-style paths are stored as variables, then reused in later `url`, `headers`, `data`, and `json` fields with `{{name}}` interpolation. The command prints JSON containing final `vars` and per-step status codes. + +`flow.yaml`: + +```yaml +steps: + - name: auth + method: POST + url: https://api.example.com/token + json: '{"client":"ci"}' + expectStatus: [200] + capture: + token: .access_token + - name: deploy + method: POST + url: https://api.example.com/deploys/{{token}} + headers: + Authorization: Bearer {{token}} + json: '{"ref":"main"}' + expectStatus: [201] + capture: + deploy_id: .id +``` + +| Flag | Description | +|---|---| +| `--header, -H` | Request header as `Name: value` | +| `--data`, `--data-file` | Raw body inline or from a file | +| `--json`, `--json-file` | JSON body inline or from a file | +| `--form, -F` | URL-encoded form field as `key=value` | +| `--file` | Multipart file field as `name=path` | +| `--expect-status` | Comma-separated acceptable status codes | +| `--jq` | Extract a jq-style path from response JSON | +| `--raw, -r` | Print extracted strings without JSON quotes | +| `--output, -o` | Write response or extracted value to a file | +| `--to-github-output` | Write response or chain JSON to `$GITHUB_OUTPUT` | +| `--retry`, `--retry-delay`, `--backoff` | Retry controls | +| `--insecure, -k` | Skip TLS certificate verification | +| `--verbose, -v` | Print status/headers or chain step statuses to stderr | + +
+ +--- + ## url ```sh @@ -1203,7 +1334,7 @@ Diagnose pipekit's runtime environment — CI platform detection, expected env v pipekit doctor # [platform] # · ci platform github-actions -# · go runtime go1.24.6 linux/amd64 +# · go runtime go1.25.11 linux/amd64 # # [tools] # ✓ git on PATH diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 022a313..ade484e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -19,7 +19,7 @@ Working on pipekit itself. For an architectural deep-dive aimed at AI assistants | Tool | Version | |---|---| -| Go | 1.24+ | +| Go | 1.25+ | | `make` | any | | `golangci-lint` | for `make lint` | | `git` | for runtime tests of `diff` / `version next` | diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 7747c87..18cd232 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -56,6 +56,88 @@ pipekit wait url http://localhost:8080/healthz --timeout 150s +
+HTTP API → JSON output without curl + jq + +```sh +# BEFORE +tag=$(curl -fsSL https://api.example.com/releases/latest | jq -r '.tag_name') +echo "tag=$tag" >> "$GITHUB_OUTPUT" + +# AFTER +pipekit http get https://api.example.com/releases/latest \ + --expect-status 200 \ + --jq .tag_name \ + --raw \ + --to-github-output tag +``` + +
+ +
+Dependent API calls without temp files + +```yaml +# flow.yaml +steps: + - name: auth + method: POST + url: https://api.example.com/token + json: '{"client":"ci"}' + capture: + token: .access_token + - name: deploy + method: POST + url: https://api.example.com/deploys/{{token}} + headers: + Authorization: Bearer {{token}} + json: '{"ref":"main"}' + expectStatus: [201] + capture: + deploy_id: .id +``` + +```sh +pipekit http chain flow.yaml --expect-status 200 --verbose +``` + +For short one-off chains, pass the plan inline: + +```sh +pipekit http chain - --expect-status 200 <<'YAML' +steps: + - name: auth + method: POST + url: https://api.example.com/token + json: '{"client":"ci"}' + capture: + token: .access_token + - name: deploy + method: POST + url: https://api.example.com/deploys/{{token}} + headers: + Authorization: Bearer {{token}} + json: '{"ref":"main"}' + expectStatus: [201] +YAML +``` + +
+ +
+Cross-platform release archive + +```sh +# BEFORE +tar --zstd -cf dist/app.tar.zst bin/app README.md + +# AFTER +pipekit archive pack dist/app.tar.zst bin/app README.md +pipekit archive list dist/app.tar.zst +``` + +
+
Retry a flaky command @@ -541,6 +623,8 @@ pipekit diff dirs --base origin/main \ make build-all pipekit artifact assert "dist/pipekit-*" +pipekit archive pack dist/release.tar.zst dist/pipekit-* README.md +pipekit archive list dist/release.tar.zst pipekit checksum files dist/pipekit-* --output dist/checksums.txt pipekit checksum verify dist/checksums.txt pipekit artifact manifest "dist/pipekit-*" "dist/checksums.txt" \ diff --git a/docs/INSTALL.md b/docs/INSTALL.md index f465ce2..a9f88db 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -83,7 +83,7 @@ Add the directory containing `pipekit.exe` to your `PATH`, or move it to a direc ## From source -Requires Go 1.24 or later. +Requires Go 1.25 or later. ```sh go install github.com/AxeForging/pipekit@latest diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 78ff168..44e6df3 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -48,7 +48,7 @@ Only relevant if you build from source. | Tool | Version | |---|---| -| Go | 1.24 or later | +| Go | 1.25 or later | | `make` | Any (used by the Makefile; not required for `go install` / `go build`) | | `golangci-lint` | For `make lint` only | diff --git a/go.mod b/go.mod index 022bf2b..c4b9fd1 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/AxeForging/pipekit -go 1.24.0 +go 1.25.0 -toolchain go1.24.6 +toolchain go1.25.11 require ( github.com/Masterminds/semver/v3 v3.4.0 @@ -10,8 +10,10 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/itchyny/gojq v0.12.18 + github.com/klauspost/compress v1.18.6 github.com/pelletier/go-toml/v2 v2.3.1 github.com/rs/zerolog v1.34.0 + github.com/ulikunitz/xz v0.5.15 github.com/urfave/cli v1.22.17 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 @@ -23,9 +25,9 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/go.sum b/go.sum index 6709e98..1627dfc 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/itchyny/gojq v0.12.18 h1:gFGHyt/MLbG9n6dqnvlliiya2TaMMh6FFaR2b1H6Drc= github.com/itchyny/gojq v0.12.18/go.mod h1:4hPoZ/3lN9fDL1D+aK7DY1f39XZpY9+1Xpjz8atrEkg= github.com/itchyny/timefmt-go v0.1.7 h1:xyftit9Tbw+Dc/huSSPJaEmX1TVL8lw5vxjJLK4GMMA= github.com/itchyny/timefmt-go v0.1.7/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -51,6 +53,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= @@ -65,15 +69,15 @@ go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 h1:h6p3mQqrmT1XkHVTfzLdNz1u7IhINeZkz67/xTbOuWs= google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= diff --git a/integration/integration_test.go b/integration/integration_test.go index 4028fc4..69accc5 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -8,6 +8,8 @@ package integration import ( "bytes" "fmt" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -484,6 +486,144 @@ func TestE2E_ChecksumAndArtifact(t *testing.T) { expectAll(t, stdout, `"path"`, `"size"`, `"sha256"`) } +func TestE2E_HTTPGetAndChain(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/release": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"tag":"v1.2.3"}`)) + case "/token": + if r.Method != http.MethodPost { + t.Fatalf("token method = %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"abc123"}`)) + case "/deploys/abc123": + if got := r.Header.Get("Authorization"); got != "Bearer abc123" { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":42}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + stdout, stderr, code := runPipekit(t, + []string{"http", "get", srv.URL + "/release", "--expect-status", "200", "--jq", ".tag", "--raw"}, "") + if code != 0 { + t.Fatalf("http get exit %d stderr=%s", code, stderr) + } + if strings.TrimSpace(stdout) != "v1.2.3" { + t.Fatalf("http get stdout = %q", stdout) + } + + dir := t.TempDir() + plan := filepath.Join(dir, "flow.yaml") + body := fmt.Sprintf(`steps: + - name: auth + method: POST + url: %s/token + json: '{"client":"ci"}' + expectStatus: [200] + capture: + token: .access_token + - name: deploy + method: POST + url: %s/deploys/{{token}} + headers: + Authorization: Bearer {{token}} + json: '{"ref":"main"}' + expectStatus: [201] + capture: + deploy_id: .id +`, srv.URL, srv.URL) + if err := os.WriteFile(plan, []byte(body), 0644); err != nil { + t.Fatal(err) + } + + // Plain file-backed chain plan. + stdout, stderr, code = runPipekit(t, + []string{"http", "chain", plan, "--expect-status", "200", "--verbose"}, "") + if code != 0 { + t.Fatalf("http chain exit %d stderr=%s stdout=%s", code, stderr, stdout) + } + expectAll(t, stdout, `"token": "abc123"`, `"deploy_id": "42"`, `"statusCode": 201`) + expectAll(t, stderr, "auth: HTTP 200", "deploy: HTTP 201") + + // Heredoc/stdin-style chain plan (`pipekit http chain - <<'YAML'`). + stdout, stderr, code = runPipekit(t, + []string{"http", "chain", "-", "--expect-status", "200"}, body) + if code != 0 { + t.Fatalf("http chain stdin exit %d stderr=%s stdout=%s", code, stderr, stdout) + } + expectAll(t, stdout, `"token": "abc123"`, `"deploy_id": "42"`, `"statusCode": 201`) +} + +func TestE2E_HTTPChainRejectsEmptyStdinPlan(t *testing.T) { + _, stderr, code := runPipekit(t, + []string{"http", "chain", "-"}, " \n") + if code == 0 { + t.Fatal("expected empty stdin plan to fail") + } + if !strings.Contains(stderr, "empty HTTP chain plan on stdin") { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestE2E_HTTPRejectsInvalidExpectedStatus(t *testing.T) { + _, stderr, code := runPipekit(t, + []string{"http", "get", "http://127.0.0.1", "--expect-status", "nope"}, "") + if code == 0 { + t.Fatal("expected invalid status to fail") + } + if !strings.Contains(stderr, "invalid status code") { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestE2E_ArchivePackListUnpack(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + if err := os.MkdirAll(filepath.Join(src, "nested"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "nested", "file.txt"), []byte("hello"), 0644); err != nil { + t.Fatal(err) + } + + archive := filepath.Join(dir, "bundle.tar.zst") + _, stderr, code := runPipekit(t, + []string{"archive", "pack", archive, src}, "") + if code != 0 { + t.Fatalf("archive pack exit %d stderr=%s", code, stderr) + } + + stdout, stderr, code := runPipekit(t, + []string{"archive", "list", archive}, "") + if code != 0 { + t.Fatalf("archive list exit %d stderr=%s", code, stderr) + } + if !strings.Contains(stdout, "src/nested/file.txt") { + t.Fatalf("archive list stdout = %q", stdout) + } + + dest := filepath.Join(dir, "out") + _, stderr, code = runPipekit(t, + []string{"archive", "unpack", archive, "--dest", dest, "--strip-components", "1"}, "") + if code != 0 { + t.Fatalf("archive unpack exit %d stderr=%s", code, stderr) + } + got, err := os.ReadFile(filepath.Join(dest, "nested", "file.txt")) + if err != nil { + t.Fatalf("read unpacked file: %v", err) + } + if string(got) != "hello" { + t.Fatalf("unpacked content = %q", got) + } +} + func TestE2E_GitAndChangelog(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not on PATH") diff --git a/main.go b/main.go index 06975b2..9748cb4 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ func main() { actions.CacheKeyCommand(), actions.ChecksumCommand(), actions.ArtifactCommand(), + actions.ArchiveCommand(), actions.GitCommand(), actions.ChangelogCommand(), actions.ConfigCommand(), @@ -49,6 +50,7 @@ func main() { actions.YAMLCommand(), actions.RenderCommand(), actions.ExecCommand(), + actions.HTTPCommand(), actions.URLCommand(), actions.ImageCommand(), actions.TimeCommand(), diff --git a/services/archive_service.go b/services/archive_service.go new file mode 100644 index 0000000..9510317 --- /dev/null +++ b/services/archive_service.go @@ -0,0 +1,505 @@ +package services + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// ArchiveEntry describes one file or directory inside an archive. +type ArchiveEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` + Mode string `json:"mode"` +} + +// PackArchive writes an archive containing the given files/directories. +func PackArchive(output string, inputs []string, format string) error { + if output == "" { + return fmt.Errorf("output archive required") + } + if len(inputs) == 0 { + return fmt.Errorf("at least one input required") + } + format = DetectArchiveFormat(output, format) + if format == "" { + return fmt.Errorf("archive format required") + } + switch format { + case "zip": + return packZip(output, inputs) + case "tar", "tar.gz", "tar.xz", "tar.zst": + return packTar(output, inputs, format) + default: + return fmt.Errorf("unsupported archive format %q", format) + } +} + +// UnpackArchive extracts an archive to dest. +func UnpackArchive(input string, dest string, format string, stripComponents int) error { + if dest == "" { + dest = "." + } + if stripComponents < 0 { + return fmt.Errorf("--strip-components cannot be negative") + } + format = DetectArchiveFormat(input, format) + switch format { + case "zip": + return unpackZip(input, dest, stripComponents) + case "tar", "tar.gz", "tar.xz", "tar.zst": + return unpackTar(input, dest, format, stripComponents) + default: + return fmt.Errorf("unsupported archive format %q", format) + } +} + +// ListArchive returns entries in an archive. +func ListArchive(input string, format string) ([]ArchiveEntry, error) { + format = DetectArchiveFormat(input, format) + switch format { + case "zip": + return listZip(input) + case "tar", "tar.gz", "tar.xz", "tar.zst": + return listTar(input, format) + default: + return nil, fmt.Errorf("unsupported archive format %q", format) + } +} + +// DetectArchiveFormat returns a normalized archive format from a flag or filename. +func DetectArchiveFormat(filename string, override string) string { + if override != "" { + return normalizeArchiveFormat(override) + } + name := strings.ToLower(filename) + switch { + case strings.HasSuffix(name, ".tar.gz"), strings.HasSuffix(name, ".tgz"): + return "tar.gz" + case strings.HasSuffix(name, ".tar.xz"), strings.HasSuffix(name, ".txz"): + return "tar.xz" + case strings.HasSuffix(name, ".tar.zst"), strings.HasSuffix(name, ".tzst"): + return "tar.zst" + case strings.HasSuffix(name, ".tar"): + return "tar" + case strings.HasSuffix(name, ".zip"): + return "zip" + default: + return "" + } +} + +func normalizeArchiveFormat(format string) string { + switch strings.ToLower(strings.TrimSpace(format)) { + case "tgz", "gz", "gzip": + return "tar.gz" + case "txz", "xz": + return "tar.xz" + case "tzst", "zst", "zstd": + return "tar.zst" + default: + return strings.ToLower(strings.TrimSpace(format)) + } +} + +func packTar(output string, inputs []string, format string) error { + out, err := os.Create(output) + if err != nil { + return fmt.Errorf("creating %s: %w", output, err) + } + defer out.Close() + + w, closeFn, err := tarWriter(out, format) + if err != nil { + return err + } + + err = walkArchiveInputs(inputs, func(path, name string, info os.FileInfo) error { + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(name) + if info.IsDir() && !strings.HasSuffix(hdr.Name, "/") { + hdr.Name += "/" + } + if err := w.WriteHeader(hdr); err != nil { + return err + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(w, f) + return err + }) + closeErr := closeFn() + if err != nil { + return err + } + return closeErr +} + +func tarWriter(out io.Writer, format string) (*tar.Writer, func() error, error) { + switch format { + case "tar": + tw := tar.NewWriter(out) + return tw, tw.Close, nil + case "tar.gz": + gw := gzip.NewWriter(out) + tw := tar.NewWriter(gw) + return tw, func() error { + if err := tw.Close(); err != nil { + return err + } + return gw.Close() + }, nil + case "tar.xz": + xw, err := xz.NewWriter(out) + if err != nil { + return nil, nil, err + } + tw := tar.NewWriter(xw) + return tw, func() error { + if err := tw.Close(); err != nil { + return err + } + return xw.Close() + }, nil + case "tar.zst": + zw, err := zstd.NewWriter(out) + if err != nil { + return nil, nil, err + } + tw := tar.NewWriter(zw) + return tw, func() error { + if err := tw.Close(); err != nil { + return err + } + zw.Close() + return nil + }, nil + default: + return nil, nil, fmt.Errorf("unsupported tar format %q", format) + } +} + +func tarReader(in io.Reader, format string) (*tar.Reader, func(), error) { + switch format { + case "tar": + return tar.NewReader(in), func() {}, nil + case "tar.gz": + gr, err := gzip.NewReader(in) + if err != nil { + return nil, nil, err + } + return tar.NewReader(gr), func() { gr.Close() }, nil + case "tar.xz": + xr, err := xz.NewReader(in) + if err != nil { + return nil, nil, err + } + return tar.NewReader(xr), func() {}, nil + case "tar.zst": + zr, err := zstd.NewReader(in) + if err != nil { + return nil, nil, err + } + return tar.NewReader(zr), func() { zr.Close() }, nil + default: + return nil, nil, fmt.Errorf("unsupported tar format %q", format) + } +} + +func packZip(output string, inputs []string) error { + out, err := os.Create(output) + if err != nil { + return fmt.Errorf("creating %s: %w", output, err) + } + defer out.Close() + + zw := zip.NewWriter(out) + + err = walkArchiveInputs(inputs, func(path, name string, info os.FileInfo) error { + hdr, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(name) + if info.IsDir() && !strings.HasSuffix(hdr.Name, "/") { + hdr.Name += "/" + } + if !info.IsDir() { + hdr.Method = zip.Deflate + } + w, err := zw.CreateHeader(hdr) + if err != nil { + return err + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(w, f) + return err + }) + closeErr := zw.Close() + if err != nil { + return err + } + return closeErr +} + +func walkArchiveInputs(inputs []string, fn func(path, name string, info os.FileInfo) error) error { + for _, input := range inputs { + info, err := os.Stat(input) + if err != nil { + return fmt.Errorf("stat %s: %w", input, err) + } + base := filepath.Dir(input) + if !info.IsDir() { + base = filepath.Dir(input) + } + if err := filepath.Walk(input, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + name, err := filepath.Rel(base, path) + if err != nil { + return err + } + name = cleanArchiveName(name) + if name == "" { + return nil + } + return fn(path, name, info) + }); err != nil { + return err + } + } + return nil +} + +func cleanArchiveName(name string) string { + name = filepath.ToSlash(filepath.Clean(name)) + name = strings.TrimPrefix(name, "./") + if name == "." || name == "/" || strings.HasPrefix(name, "../") || name == ".." { + return "" + } + return strings.TrimPrefix(name, "/") +} + +func unpackTar(input string, dest string, format string, stripComponents int) error { + f, err := os.Open(input) + if err != nil { + return fmt.Errorf("opening %s: %w", input, err) + } + defer f.Close() + + tr, closeFn, err := tarReader(f, format) + if err != nil { + return err + } + defer closeFn() + + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + name, ok, err := stripArchiveName(hdr.Name, stripComponents) + if err != nil { + return err + } + if !ok { + continue + } + target, err := safeArchiveTarget(dest, name) + if err != nil { + return err + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + _, copyErr := io.Copy(out, tr) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } + } +} + +func unpackZip(input string, dest string, stripComponents int) error { + zr, err := zip.OpenReader(input) + if err != nil { + return fmt.Errorf("opening %s: %w", input, err) + } + defer zr.Close() + + for _, file := range zr.File { + name, ok, err := stripArchiveName(file.Name, stripComponents) + if err != nil { + return err + } + if !ok { + continue + } + target, err := safeArchiveTarget(dest, name) + if err != nil { + return err + } + if file.FileInfo().IsDir() { + if err := os.MkdirAll(target, file.Mode()); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + rc, err := file.Open() + if err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, file.Mode()) + if err != nil { + rc.Close() + return err + } + _, copyErr := io.Copy(out, rc) + closeErr := out.Close() + rcErr := rc.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if rcErr != nil { + return rcErr + } + } + return nil +} + +func listTar(input string, format string) ([]ArchiveEntry, error) { + f, err := os.Open(input) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", input, err) + } + defer f.Close() + tr, closeFn, err := tarReader(f, format) + if err != nil { + return nil, err + } + defer closeFn() + + var entries []ArchiveEntry + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + entries = append(entries, ArchiveEntry{Name: hdr.Name, Size: hdr.Size, Mode: os.FileMode(hdr.Mode).String()}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries, nil +} + +func listZip(input string) ([]ArchiveEntry, error) { + zr, err := zip.OpenReader(input) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", input, err) + } + defer zr.Close() + entries := make([]ArchiveEntry, 0, len(zr.File)) + for _, file := range zr.File { + entries = append(entries, ArchiveEntry{Name: file.Name, Size: int64(file.UncompressedSize64), Mode: file.Mode().String()}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries, nil +} + +func stripArchiveName(name string, components int) (string, bool, error) { + name, err := validateArchiveEntryName(name) + if err != nil { + return "", false, err + } + if name == "" { + return "", false, nil + } + parts := strings.Split(name, "/") + if len(parts) <= components { + return "", false, nil + } + return strings.Join(parts[components:], "/"), true, nil +} + +func validateArchiveEntryName(name string) (string, error) { + name = filepath.ToSlash(strings.TrimSpace(name)) + name = strings.TrimPrefix(name, "./") + if name == "" || name == "." { + return "", nil + } + if strings.HasPrefix(name, "/") { + return "", fmt.Errorf("archive entry %q escapes destination", name) + } + for _, part := range strings.Split(name, "/") { + if part == ".." { + return "", fmt.Errorf("archive entry %q escapes destination", name) + } + } + return filepath.ToSlash(filepath.Clean(name)), nil +} + +func safeArchiveTarget(dest, name string) (string, error) { + cleanDest, err := filepath.Abs(dest) + if err != nil { + return "", err + } + target, err := filepath.Abs(filepath.Join(cleanDest, filepath.FromSlash(name))) + if err != nil { + return "", err + } + if target != cleanDest && !strings.HasPrefix(target, cleanDest+string(os.PathSeparator)) { + return "", fmt.Errorf("archive entry %q escapes destination", name) + } + return target, nil +} diff --git a/services/archive_service_test.go b/services/archive_service_test.go new file mode 100644 index 0000000..0929deb --- /dev/null +++ b/services/archive_service_test.go @@ -0,0 +1,116 @@ +package services + +import ( + "archive/tar" + "archive/zip" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestArchiveRoundTripFormats(t *testing.T) { + formats := map[string]string{ + "zip": "bundle.zip", + "tar": "bundle.tar", + "tar.gz": "bundle.tar.gz", + "tar.xz": "bundle.tar.xz", + "tar.zst": "bundle.tar.zst", + } + for format, name := range formats { + t.Run(format, func(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + if err := os.MkdirAll(filepath.Join(src, "nested"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "nested", "file.txt"), []byte("hello"), 0644); err != nil { + t.Fatal(err) + } + + archive := filepath.Join(dir, name) + if err := PackArchive(archive, []string{src}, format); err != nil { + t.Fatalf("pack: %v", err) + } + entries, err := ListArchive(archive, "") + if err != nil { + t.Fatalf("list: %v", err) + } + if !archiveHasEntry(entries, "src/nested/file.txt") { + t.Fatalf("entries missing file: %#v", entries) + } + + dest := filepath.Join(dir, "out") + if err := UnpackArchive(archive, dest, "", 1); err != nil { + t.Fatalf("unpack: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "nested", "file.txt")) + if err != nil { + t.Fatalf("read unpacked: %v", err) + } + if string(got) != "hello" { + t.Fatalf("content = %q", got) + } + }) + } +} + +func TestArchiveRejectsZipTraversal(t *testing.T) { + dir := t.TempDir() + archive := filepath.Join(dir, "bad.zip") + out, err := os.Create(archive) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(out) + w, err := zw.Create("../escape.txt") + if err != nil { + t.Fatal(err) + } + _, _ = w.Write([]byte("bad")) + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + + err = UnpackArchive(archive, filepath.Join(dir, "out"), "", 0) + if err == nil || !strings.Contains(err.Error(), "escapes destination") { + t.Fatalf("expected traversal error, got %v", err) + } +} + +func TestArchiveRejectsTarTraversal(t *testing.T) { + dir := t.TempDir() + archive := filepath.Join(dir, "bad.tar") + out, err := os.Create(archive) + if err != nil { + t.Fatal(err) + } + tw := tar.NewWriter(out) + if err := tw.WriteHeader(&tar.Header{Name: "../escape.txt", Mode: 0644, Size: 3}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte("bad")) + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + + err = UnpackArchive(archive, filepath.Join(dir, "out"), "", 0) + if err == nil || !strings.Contains(err.Error(), "escapes destination") { + t.Fatalf("expected traversal error, got %v", err) + } +} + +func archiveHasEntry(entries []ArchiveEntry, name string) bool { + for _, entry := range entries { + if entry.Name == name { + return true + } + } + return false +} diff --git a/services/http_service.go b/services/http_service.go new file mode 100644 index 0000000..b2b6d24 --- /dev/null +++ b/services/http_service.go @@ -0,0 +1,355 @@ +package services + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// HTTPRequestOptions configures one HTTP request. +type HTTPRequestOptions struct { + Method string + URL string + Headers map[string]string + Body []byte + Timeout time.Duration + ExpectedStatus []int + RetryAttempts int + RetryDelay time.Duration + Backoff bool + InsecureTLS bool +} + +// HTTPResult contains a completed HTTP response. +type HTTPResult struct { + StatusCode int + Headers http.Header + Body []byte +} + +// ExecuteHTTPRequest runs an HTTP request, optionally retrying failed attempts. +func ExecuteHTTPRequest(opts HTTPRequestOptions) (*HTTPResult, error) { + if opts.Method == "" { + opts.Method = http.MethodGet + } + if opts.Timeout <= 0 { + opts.Timeout = 30 * time.Second + } + if opts.RetryAttempts <= 0 { + opts.RetryAttempts = 1 + } + delay := opts.RetryDelay + if delay <= 0 { + delay = time.Second + } + + var lastErr error + for attempt := 1; attempt <= opts.RetryAttempts; attempt++ { + res, err := doHTTPRequest(opts) + if err == nil { + return res, nil + } + lastErr = err + if attempt == opts.RetryAttempts { + break + } + time.Sleep(delay) + if opts.Backoff { + delay *= 2 + } + } + return nil, lastErr +} + +func doHTTPRequest(opts HTTPRequestOptions) (*HTTPResult, error) { + req, err := http.NewRequest(strings.ToUpper(opts.Method), opts.URL, bytes.NewReader(opts.Body)) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + for k, v := range opts.Headers { + req.Header.Set(k, v) + } + + client := &http.Client{Timeout: opts.Timeout} + if opts.InsecureTLS { + client.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("requesting %s: %w", opts.URL, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + + result := &HTTPResult{StatusCode: resp.StatusCode, Headers: resp.Header.Clone(), Body: body} + if len(opts.ExpectedStatus) > 0 && !statusAllowed(resp.StatusCode, opts.ExpectedStatus) { + return result, fmt.Errorf("HTTP %s returned status %d, expected one of %v", opts.URL, resp.StatusCode, opts.ExpectedStatus) + } + return result, nil +} + +// HTTPChainPlan defines a sequence of dependent HTTP requests. +type HTTPChainPlan struct { + Steps []HTTPChainStep `json:"steps"` +} + +// HTTPChainStep defines one request in an HTTP chain. +type HTTPChainStep struct { + Name string `json:"name"` + Method string `json:"method"` + URL string `json:"url"` + Headers map[string]string `json:"headers"` + Data string `json:"data"` + JSON string `json:"json"` + ExpectStatus []int `json:"expectStatus"` + Capture map[string]string `json:"capture"` + TimeoutSeconds int `json:"timeoutSeconds"` +} + +// HTTPChainResult is the structured output from a chain run. +type HTTPChainResult struct { + Vars map[string]string `json:"vars"` + Steps []HTTPChainStepResult `json:"steps"` +} + +// HTTPChainStepResult records one step outcome. +type HTTPChainStepResult struct { + Name string `json:"name"` + StatusCode int `json:"statusCode"` +} + +// DecodeHTTPChainPlan converts decoded JSON/YAML/TOML data into a chain plan. +func DecodeHTTPChainPlan(v interface{}) (HTTPChainPlan, error) { + b, err := json.Marshal(v) + if err != nil { + return HTTPChainPlan{}, err + } + var plan HTTPChainPlan + if err := json.Unmarshal(b, &plan); err != nil { + return HTTPChainPlan{}, fmt.Errorf("decoding HTTP chain plan: %w", err) + } + if len(plan.Steps) == 0 { + return HTTPChainPlan{}, fmt.Errorf("HTTP chain requires at least one step") + } + return plan, nil +} + +// ExecuteHTTPChain runs a dependent sequence of HTTP requests. +func ExecuteHTTPChain(plan HTTPChainPlan, base HTTPRequestOptions) (*HTTPChainResult, error) { + vars := map[string]string{} + result := &HTTPChainResult{Vars: vars} + + for i, step := range plan.Steps { + name := step.Name + if name == "" { + name = fmt.Sprintf("step%d", i+1) + } + method := step.Method + if method == "" { + method = http.MethodGet + } + headers := make(map[string]string, len(base.Headers)+len(step.Headers)) + for k, v := range base.Headers { + headers[k] = interpolateHTTPVars(v, vars) + } + for k, v := range step.Headers { + headers[k] = interpolateHTTPVars(v, vars) + } + body, contentType, err := chainStepBody(step, vars) + if err != nil { + return result, fmt.Errorf("%s: %w", name, err) + } + if contentType != "" { + if _, exists := headers["Content-Type"]; !exists { + headers["Content-Type"] = contentType + } + } + timeout := base.Timeout + if step.TimeoutSeconds > 0 { + timeout = time.Duration(step.TimeoutSeconds) * time.Second + } + expected := step.ExpectStatus + if len(expected) == 0 { + expected = base.ExpectedStatus + } + + res, err := ExecuteHTTPRequest(HTTPRequestOptions{ + Method: method, + URL: interpolateHTTPVars(step.URL, vars), + Headers: headers, + Body: body, + Timeout: timeout, + ExpectedStatus: expected, + RetryAttempts: base.RetryAttempts, + RetryDelay: base.RetryDelay, + Backoff: base.Backoff, + InsecureTLS: base.InsecureTLS, + }) + if err != nil { + return result, fmt.Errorf("%s: %w", name, err) + } + result.Steps = append(result.Steps, HTTPChainStepResult{Name: name, StatusCode: res.StatusCode}) + for key, path := range step.Capture { + val, err := ExtractHTTPJSON(res.Body, path) + if err != nil { + return result, fmt.Errorf("%s capture %s: %w", name, key, err) + } + vars[key] = fmt.Sprint(val) + } + } + return result, nil +} + +func chainStepBody(step HTTPChainStep, vars map[string]string) ([]byte, string, error) { + if step.Data != "" && step.JSON != "" { + return nil, "", fmt.Errorf("use only one of data or json") + } + if step.Data != "" { + return []byte(interpolateHTTPVars(step.Data, vars)), "", nil + } + if step.JSON != "" { + body := []byte(interpolateHTTPVars(step.JSON, vars)) + if !json.Valid(body) { + return nil, "", fmt.Errorf("json body is not valid JSON after interpolation") + } + return body, "application/json", nil + } + return nil, "", nil +} + +func interpolateHTTPVars(value string, vars map[string]string) string { + out := value + for k, v := range vars { + out = strings.ReplaceAll(out, "{{"+k+"}}", v) + } + return out +} + +func statusAllowed(got int, expected []int) bool { + for _, code := range expected { + if got == code { + return true + } + } + return false +} + +// ExtractHTTPJSON parses response JSON and returns the value at a jq-style path. +func ExtractHTTPJSON(body []byte, path string) (interface{}, error) { + var doc interface{} + if err := json.Unmarshal(body, &doc); err != nil { + return nil, fmt.Errorf("decoding response JSON: %w", err) + } + return JSONGet(doc, path) +} + +// ParseHTTPHeaders parses repeated "Name: value" header flags. +func ParseHTTPHeaders(headers []string) (map[string]string, error) { + out := make(map[string]string, len(headers)) + for _, h := range headers { + name, value, ok := strings.Cut(h, ":") + if !ok || strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("invalid header %q, expected 'Name: value'", h) + } + out[strings.TrimSpace(name)] = strings.TrimSpace(value) + } + return out, nil +} + +// BuildHTTPBody builds a request body from data, JSON, file, or form inputs. +func BuildHTTPBody(data string, dataFile string, jsonValue string, jsonFile string, formFields []string) ([]byte, string, error) { + set := 0 + for _, v := range []string{data, dataFile, jsonValue, jsonFile} { + if v != "" { + set++ + } + } + if len(formFields) > 0 { + set++ + } + if set > 1 { + return nil, "", fmt.Errorf("use only one of --data, --data-file, --json, --json-file, or --form") + } + + switch { + case data != "": + return []byte(data), "", nil + case dataFile != "": + b, err := os.ReadFile(dataFile) + if err != nil { + return nil, "", fmt.Errorf("reading %s: %w", dataFile, err) + } + return b, "", nil + case jsonValue != "": + if !json.Valid([]byte(jsonValue)) { + return nil, "", fmt.Errorf("--json is not valid JSON") + } + return []byte(jsonValue), "application/json", nil + case jsonFile != "": + b, err := os.ReadFile(jsonFile) + if err != nil { + return nil, "", fmt.Errorf("reading %s: %w", jsonFile, err) + } + if !json.Valid(b) { + return nil, "", fmt.Errorf("%s is not valid JSON", jsonFile) + } + return b, "application/json", nil + case len(formFields) > 0: + values := url.Values{} + for _, field := range formFields { + k, v, ok := strings.Cut(field, "=") + if !ok || k == "" { + return nil, "", fmt.Errorf("invalid form field %q, expected key=value", field) + } + values.Add(k, v) + } + return []byte(values.Encode()), "application/x-www-form-urlencoded", nil + default: + return nil, "", nil + } +} + +// BuildMultipartBody builds a multipart/form-data request body from key=path pairs. +func BuildMultipartBody(fileFields []string) ([]byte, string, error) { + if len(fileFields) == 0 { + return nil, "", nil + } + var b bytes.Buffer + w := multipart.NewWriter(&b) + for _, field := range fileFields { + name, path, ok := strings.Cut(field, "=") + if !ok || name == "" || path == "" { + return nil, "", fmt.Errorf("invalid file field %q, expected name=path", field) + } + f, err := os.Open(path) + if err != nil { + return nil, "", fmt.Errorf("opening %s: %w", path, err) + } + part, err := w.CreateFormFile(name, path) + if err != nil { + f.Close() + return nil, "", err + } + if _, err := io.Copy(part, f); err != nil { + f.Close() + return nil, "", err + } + f.Close() + } + if err := w.Close(); err != nil { + return nil, "", err + } + return b.Bytes(), w.FormDataContentType(), nil +} diff --git a/services/http_service_test.go b/services/http_service_test.go new file mode 100644 index 0000000..2154303 --- /dev/null +++ b/services/http_service_test.go @@ -0,0 +1,139 @@ +package services + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestExecuteHTTPRequestExtractJSON(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Test"); got != "yes" { + t.Fatalf("header = %q", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"release":{"tag":"v1.2.3"}}`)) + })) + defer srv.Close() + + res, err := ExecuteHTTPRequest(HTTPRequestOptions{ + Method: http.MethodGet, + URL: srv.URL, + Headers: map[string]string{"X-Test": "yes"}, + Timeout: 2 * time.Second, + ExpectedStatus: []int{200}, + }) + if err != nil { + t.Fatalf("request: %v", err) + } + val, err := ExtractHTTPJSON(res.Body, ".release.tag") + if err != nil { + t.Fatalf("extract: %v", err) + } + if val != "v1.2.3" { + t.Fatalf("tag = %v", val) + } +} + +func TestExecuteHTTPRequestRetriesStatusFailures(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusCreated) + })) + defer srv.Close() + + res, err := ExecuteHTTPRequest(HTTPRequestOptions{ + Method: http.MethodPost, + URL: srv.URL, + Timeout: 2 * time.Second, + ExpectedStatus: []int{201}, + RetryAttempts: 3, + RetryDelay: time.Millisecond, + }) + if err != nil { + t.Fatalf("request: %v", err) + } + if res.StatusCode != http.StatusCreated { + t.Fatalf("status = %d", res.StatusCode) + } + if calls.Load() != 3 { + t.Fatalf("calls = %d", calls.Load()) + } +} + +func TestBuildHTTPBodyModes(t *testing.T) { + body, contentType, err := BuildHTTPBody("", "", `{"ok":true}`, "", nil) + if err != nil { + t.Fatalf("json body: %v", err) + } + if contentType != "application/json" || !json.Valid(body) { + t.Fatalf("unexpected json body: %s %s", contentType, body) + } + + body, contentType, err = BuildHTTPBody("", "", "", "", []string{"a=1", "b=two words"}) + if err != nil { + t.Fatalf("form body: %v", err) + } + if contentType != "application/x-www-form-urlencoded" || string(body) != "a=1&b=two+words" { + t.Fatalf("unexpected form body: %s %s", contentType, body) + } +} + +func TestExecuteHTTPChainCapturesAndInterpolates(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + _, _ = w.Write([]byte(`{"token":"abc123"}`)) + case "/items/abc123": + if got := r.Header.Get("Authorization"); got != "Bearer abc123" { + t.Fatalf("Authorization = %q", got) + } + _, _ = w.Write([]byte(`{"id":42}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + result, err := ExecuteHTTPChain(HTTPChainPlan{Steps: []HTTPChainStep{ + { + Name: "auth", + Method: "GET", + URL: srv.URL + "/token", + ExpectStatus: []int{200}, + Capture: map[string]string{"token": ".token"}, + }, + { + Name: "item", + Method: "GET", + URL: srv.URL + "/items/{{token}}", + Headers: map[string]string{"Authorization": "Bearer {{token}}"}, + ExpectStatus: []int{200}, + Capture: map[string]string{"item_id": ".id"}, + }, + }}, HTTPRequestOptions{Timeout: 2 * time.Second}) + if err != nil { + t.Fatalf("chain: %v", err) + } + if result.Vars["token"] != "abc123" || result.Vars["item_id"] != "42" { + t.Fatalf("vars = %#v", result.Vars) + } + if len(result.Steps) != 2 { + t.Fatalf("steps = %d", len(result.Steps)) + } +} + +func TestParseHTTPHeadersRejectsInvalid(t *testing.T) { + _, err := ParseHTTPHeaders([]string{"not a header"}) + if err == nil || !strings.Contains(err.Error(), "invalid header") { + t.Fatalf("expected invalid header error, got %v", err) + } +}