feat(app): resume installed apps on stack up + persisted value overrides + sell linkage

README.md

@@ -336,6 +336,10 @@ obol app install bitnami/redis
 # With specific version
 obol app install bitnami/postgresql@15.0.0
 
+# Customize values at install or sync time (persisted into the app's values.yaml)
+obol app install bitnami/redis --set architecture=standalone --set auth.enabled=false
+obol app sync redis --values ./my-overrides.yaml --set image.tag=7.2
+
 # Deploy to cluster (auto-selects if only one app is installed)
 obol app sync
 obol app sync postgresql/eager-fox
@@ -345,6 +349,8 @@ obol app list
 obol app delete postgresql/eager-fox --force
 ```
 
+Installed apps are re-synced automatically on `obol stack up`, so they survive cluster recreation. After a sync, the CLI prints a ready-to-run `obol sell http` command for each service the app exposes.
+
 Find charts at [Artifact Hub](https://artifacthub.io).
 
 ## Managing the Stack

cmd/obol/main.go

@@ -223,6 +223,14 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}}
 							// Best-effort. No-op when `obol sell info set` was
 							// never used.
 							storefront.ReconcileRecorded(cfg, u)
+							// Re-sync installed app deployments BEFORE sell
+							// offers: `obol sell http` offers can gate an
+							// app's Service as their upstream, and the
+							// controller's upstream health check needs it
+							// present. App state is declarative on disk
+							// (helmfile.yaml + values.yaml) but its cluster
+							// resources live in etcd. Best-effort.
+							app.ResumeAll(cfg, u)
 							// Re-apply cluster-side state for locally-persisted
 							// `obol sell *` offers. ServiceOffer CRs and the
 							// Service/Endpoints that route to the host gateway
@@ -373,6 +381,14 @@ Find charts at https://artifacthub.io`,
 								Aliases: []string{"f"},
 								Usage:   "Overwrite existing deployment",
 							},
+							&cli.StringSliceFlag{
+								Name:  "values",
+								Usage: "Values file merged onto chart defaults (repeatable, merged in order)",
+							},
+							&cli.StringSliceFlag{
+								Name:  "set",
+								Usage: "Override a value, e.g. --set image.tag=1.2.3 (repeatable, applied after --values)",
+							},
 						},
 						Action: func(ctx context.Context, cmd *cli.Command) error {
 							if cmd.NArg() == 0 {
@@ -387,10 +403,12 @@ Find charts at https://artifacthub.io`,
 
 							chartRef := cmd.Args().First()
 							opts := app.InstallOptions{
-								Name:    cmd.String("name"),
-								Version: cmd.String("version"),
-								ID:      cmd.String("id"),
-								Force:   cmd.Bool("force"),
+								Name:        cmd.String("name"),
+								Version:     cmd.String("version"),
+								ID:          cmd.String("id"),
+								Force:       cmd.Bool("force"),
+								ValuesFiles: cmd.StringSlice("values"),
+								Set:         cmd.StringSlice("set"),
 							}
 
 							return app.Install(cfg, getUI(cmd), chartRef, opts)
@@ -400,13 +418,37 @@ Find charts at https://artifacthub.io`,
 						Name:      "sync",
 						Usage:     "Deploy application to cluster",
 						ArgsUsage: "[<app>/<id>]",
+						Flags: []cli.Flag{
+							&cli.StringSliceFlag{
+								Name:  "values",
+								Usage: "Values file merged into the deployment's values.yaml before syncing (repeatable)",
+							},
+							&cli.StringSliceFlag{
+								Name:  "set",
+								Usage: "Override a value before syncing, e.g. --set image.tag=1.2.3 (repeatable)",
+							},
+						},
 						Action: func(ctx context.Context, cmd *cli.Command) error {
 							identifier, _, err := app.ResolveInstance(cfg, cmd.Args().Slice())
 							if err != nil {
 								return err
 							}
 
-							return app.Sync(cfg, getUI(cmd), identifier)
+							u := getUI(cmd)
+							// Overrides are persisted into the deployment's
+							// values.yaml (not passed as ephemeral flags) so
+							// resume-on-stack-up replays them.
+							if err := app.ApplyOverrides(cfg, identifier, cmd.StringSlice("values"), cmd.StringSlice("set")); err != nil {
+								return err
+							}
+
+							if err := app.Sync(cfg, u, identifier); err != nil {
+								return err
+							}
+
+							app.PrintSellHint(cfg, u, identifier)
+
+							return nil
 						},
 					},
 					{

cmd/obol/sell.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/ObolNetwork/obol-stack/internal/agentcrd"
+	"github.com/ObolNetwork/obol-stack/internal/app"
 	"github.com/ObolNetwork/obol-stack/internal/config"
 	"github.com/ObolNetwork/obol-stack/internal/erc8004"
 	"github.com/ObolNetwork/obol-stack/internal/hermes"
@@ -3943,6 +3944,10 @@ Examples:
 			// Recorded Agent CRs first: agent-backed offers resolve
 			// agent.ref and would dangle on a freshly-recreated cluster.
 			agentcrd.ResumeAll(cfg, u)
+			// Installed apps next: http offers can gate an app's Service
+			// as their upstream, so the Service must exist before the
+			// offer republishes. Best-effort.
+			app.ResumeAll(cfg, u)
 			if err := resumeSellOffers(ctx, cfg, u); err != nil {
 				return err
 			}

cmd/obol/stackup_resume_guard_test.go

@@ -22,6 +22,7 @@ func TestStackUpAction_ReplaysRecordedState(t *testing.T) {
 	upIdx := strings.Index(body, "stack.Up(cfg")
 	rpcIdx := strings.Index(body, "network.ReconcileRecordedRPCs(")
 	agentsIdx := strings.Index(body, "agentcrd.ResumeAll(")
+	appsIdx := strings.Index(body, "app.ResumeAll(")
 	offersIdx := strings.Index(body, "resumeSellOffers(")
 
 	if rpcIdx < 0 {
@@ -30,15 +31,21 @@ func TestStackUpAction_ReplaysRecordedState(t *testing.T) {
 	if agentsIdx < 0 {
 		t.Fatal("cmd/obol/main.go must call agentcrd.ResumeAll — without it recorded Agent CRs never reach a freshly-recreated cluster")
 	}
+	if appsIdx < 0 {
+		t.Fatal("cmd/obol/main.go must call app.ResumeAll — without it installed apps never reach a freshly-recreated cluster")
+	}
 	if upIdx < 0 || offersIdx < 0 {
 		t.Fatalf("expected stack.Up and resumeSellOffers in main.go; upIdx=%d offersIdx=%d", upIdx, offersIdx)
 	}
-	if rpcIdx < upIdx || agentsIdx < upIdx {
+	if rpcIdx < upIdx || agentsIdx < upIdx || appsIdx < upIdx {
 		t.Error("recorded-state replay must run AFTER stack.Up — before it there is no kubeconfig/cluster")
 	}
 	if agentsIdx > offersIdx {
 		t.Error("agentcrd.ResumeAll must run BEFORE resumeSellOffers — agent-backed ServiceOffers need their Agent CR first")
 	}
+	if appsIdx > offersIdx {
+		t.Error("app.ResumeAll must run BEFORE resumeSellOffers — http ServiceOffers can gate an app's Service as their upstream")
+	}
 }
 
 // TestSellResumeAction_ReplaysAgentsBeforeOffers extends the same guard to
@@ -56,6 +63,10 @@ func TestSellResumeAction_ReplaysAgentsBeforeOffers(t *testing.T) {
 	if agentsIdx < 0 {
 		t.Fatal("cmd/obol/sell.go (sell resume action) must call agentcrd.ResumeAll before replaying offers")
 	}
+	appsIdx := strings.Index(body, "app.ResumeAll(")
+	if appsIdx < 0 {
+		t.Fatal("cmd/obol/sell.go (sell resume action) must call app.ResumeAll before replaying offers — http offers can gate app upstreams")
+	}
 	// The resume action's offer replay is the only call site that returns
 	// the error (`if err := resumeSellOffers(...)`); main.go's stack-up
 	// call warns instead.
@@ -66,4 +77,7 @@ func TestSellResumeAction_ReplaysAgentsBeforeOffers(t *testing.T) {
 	if agentsIdx > offersIdx {
 		t.Error("agentcrd.ResumeAll must run BEFORE resumeSellOffers in the sell resume action")
 	}
+	if appsIdx > offersIdx {
+		t.Error("app.ResumeAll must run BEFORE resumeSellOffers in the sell resume action")
+	}
 }

internal/app/app.go

@@ -18,10 +18,12 @@ import (
 
 // InstallOptions contains options for the install command
 type InstallOptions struct {
-	Name    string // Optional app name override
-	Version string // Chart version (empty = latest for repo/chart, extracted for URL)
-	ID      string // Deployment ID (empty = generate petname)
-	Force   bool   // Overwrite existing deployment
+	Name        string   // Optional app name override
+	Version     string   // Chart version (empty = latest for repo/chart, extracted for URL)
+	ID          string   // Deployment ID (empty = generate petname)
+	Force       bool     // Overwrite existing deployment
+	ValuesFiles []string // Values files merged onto chart defaults (in order)
+	Set         []string // key.path=value overrides applied after values files
 }
 
 // ListOptions contains options for the list command
@@ -117,13 +119,20 @@ func Install(cfg *config.Config, u *ui.UI, chartRef string, opts InstallOptions)
 		return fmt.Errorf("failed to write values.yaml: %w", err)
 	}
 
-	// 9. Generate helmfile.yaml (references chart remotely)
+	// 9. Merge user overrides (--values files, then --set) into values.yaml
+	// so the deployment directory records exactly what will deploy.
+	if err := applyOverridesToFile(valuesPath, opts.ValuesFiles, opts.Set); err != nil {
+		os.RemoveAll(deploymentDir)
+		return fmt.Errorf("failed to apply value overrides: %w", err)
+	}
+
+	// 10. Generate helmfile.yaml (references chart remotely)
 	if err := generateRemoteHelmfile(deploymentDir, chart, appName, id); err != nil {
 		os.RemoveAll(deploymentDir)
 		return fmt.Errorf("failed to generate helmfile: %w", err)
 	}
 
-	// 10. Print success message
+	// 11. Print success message
 	u.Blank()
 	u.Successf("Application installed successfully!")
 	u.Detail("Deployment", fmt.Sprintf("%s/%s", appName, id))

internal/app/resume.go

@@ -0,0 +1,36 @@
+package app
+
+import (
+	"github.com/ObolNetwork/obol-stack/internal/config"
+	"github.com/ObolNetwork/obol-stack/internal/ui"
+)
+
+// ResumeAll re-syncs every installed app deployment into the (possibly
+// freshly recreated) cluster. App deployments are declarative on disk
+// (helmfile.yaml + values.yaml under applications/<app>/<id>/) but their
+// cluster resources live in etcd, which `obol stack down` + recreate
+// destroys — without this replay a fresh `stack up` comes back with the
+// deployment directories still on disk but no matching cluster resources,
+// and any `obol sell http` offer gating an app upstream dangles.
+//
+// Best-effort, mirroring agentcrd.ResumeAll: a failed app warns and does
+// not block stack-up.
+func ResumeAll(cfg *config.Config, u *ui.UI) {
+	ids, err := ListInstanceIDs(cfg)
+	if err != nil {
+		u.Warnf("Could not list installed apps: %v", err)
+		return
+	}
+
+	if len(ids) == 0 {
+		return
+	}
+
+	u.Infof("Re-syncing %d installed app(s)...", len(ids))
+
+	for _, id := range ids {
+		if err := Sync(cfg, u, id); err != nil {
+			u.Warnf("Could not re-sync app %s (run 'obol app sync %s' to retry): %v", id, id, err)
+		}
+	}
+}

internal/app/resume_test.go

@@ -0,0 +1,63 @@
+package app
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ObolNetwork/obol-stack/internal/config"
+	"github.com/ObolNetwork/obol-stack/internal/ui"
+)
+
+// TestResumeAll_NoApps ensures a config dir with no app deployments is a
+// silent no-op — stack up must not print resume noise for users who never
+// installed an app.
+func TestResumeAll_NoApps(t *testing.T) {
+	cfg := &config.Config{ConfigDir: t.TempDir()}
+
+	var stdout, stderr bytes.Buffer
+	ResumeAll(cfg, ui.NewForTest(&stdout, &stderr))
+
+	if out := stdout.String() + stderr.String(); out != "" {
+		t.Errorf("expected no output with zero apps, got: %q", out)
+	}
+}
+
+// TestResumeAll_SkipsNonAppStateDirs ensures resume only replays real app
+// deployments (identified by values.yaml) and never touches sibling
+// subsystem state dirs like applications/hermes/<id>, which use
+// values-<component>.yaml naming and are NOT helmfile-syncable apps.
+func TestResumeAll_SkipsNonAppStateDirs(t *testing.T) {
+	configDir := t.TempDir()
+	cfg := &config.Config{ConfigDir: configDir}
+
+	appDir := filepath.Join(configDir, "applications", "redis", "eager-fox")
+	if err := os.MkdirAll(appDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeFile(t, appDir, "values.yaml", "a: 1\n")
+	writeFile(t, appDir, "helmfile.yaml", "releases: []\n")
+
+	hermesDir := filepath.Join(configDir, "applications", "hermes", "obol-agent")
+	if err := os.MkdirAll(hermesDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeFile(t, hermesDir, "values-hermes.yaml", "b: 2\n")
+	writeFile(t, hermesDir, "helmfile.yaml", "releases: []\n")
+
+	var stdout, stderr bytes.Buffer
+	// No kubeconfig.yaml in configDir, so Sync fails fast with "cluster
+	// not running" — the point is WHICH deployments resume attempts.
+	ResumeAll(cfg, ui.NewForTest(&stdout, &stderr))
+
+	out := stdout.String() + stderr.String()
+
+	if !strings.Contains(out, "redis/eager-fox") {
+		t.Errorf("expected resume attempt for redis/eager-fox, got: %q", out)
+	}
+	if strings.Contains(out, "hermes") {
+		t.Errorf("resume must not touch the hermes state dir, got: %q", out)
+	}
+}

internal/app/sellhint.go

@@ -0,0 +1,75 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/ObolNetwork/obol-stack/internal/config"
+	"github.com/ObolNetwork/obol-stack/internal/ui"
+)
+
+// maxSellHints caps how many per-service sell suggestions are printed
+// after a sync; charts with many Services (umbrella charts) would
+// otherwise drown the success output.
+const maxSellHints = 3
+
+// PrintSellHint discovers the Services a just-synced app exposes and
+// prints a ready-to-run `obol sell http` command for each, connecting app
+// installs to the monetization flow. Best-effort: any discovery failure
+// (no kubeconfig, kubectl missing, no Services yet) prints nothing.
+func PrintSellHint(cfg *config.Config, u *ui.UI, deploymentIdentifier string) {
+	appName, id, err := parseDeploymentIdentifier(deploymentIdentifier)
+	if err != nil {
+		return
+	}
+
+	namespace := fmt.Sprintf("%s-%s", appName, id)
+
+	kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml")
+	if _, err := os.Stat(kubeconfigPath); err != nil {
+		return
+	}
+
+	kubectlBinary := filepath.Join(cfg.BinDir, "kubectl")
+	if _, err := os.Stat(kubectlBinary); err != nil {
+		return
+	}
+
+	cmd := exec.Command(kubectlBinary, "get", "services", "-n", namespace,
+		"-o", `jsonpath={range .items[*]}{.metadata.name} {.spec.ports[0].port}{"\n"}{end}`)
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+kubeconfigPath)
+
+	out, err := cmd.Output()
+	if err != nil {
+		return
+	}
+
+	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+
+	printed := 0
+
+	for _, line := range lines {
+		name, port, ok := strings.Cut(strings.TrimSpace(line), " ")
+		if !ok || name == "" || port == "" {
+			continue
+		}
+
+		if printed == 0 {
+			u.Blank()
+			u.Print("To put this app on sale behind x402 payments:")
+		}
+
+		if printed == maxSellHints {
+			u.Printf("  (more services: obol kubectl get svc -n %s)", namespace)
+			break
+		}
+
+		u.Printf("  obol sell http %s --upstream %s --port %s --namespace %s --per-request 0.001 --pay-to 0x<wallet>",
+			name, name, port, namespace)
+
+		printed++
+	}
+}