From 6dbc4e0d011b7f721d8b48d759d50c28d9ef27f9 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:21:29 -0700 Subject: [PATCH 01/70] test: make [[ ]] assertions effective under bash 3.2 (EGB-677 precursor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bats on macOS runs under system bash 3.2, where a failing [[ ]] compound command mid-test does NOT trip the ERR trap — every mid-test [[ "$output" == *...* ]] assertion in the suite was decorative. Append '|| false' to all 123 standalone [[ ]] assertion lines so failures become plain-command failures, which bats catches. Full suite still green (133/133) — no latent false positives were hiding. --- test/secrets.bats | 246 +++++++++++++++++++++++----------------------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/test/secrets.bats b/test/secrets.bats index f396171..56f051e 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -27,7 +27,7 @@ load test_helper run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"Already initialized"* ]] + [[ "$output" == *"Already initialized"* ]] || false # Key must not be overwritten local key_after @@ -51,7 +51,7 @@ load test_helper run env PATH="$fake_path" "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"age"* ]] + [[ "$output" == *"age"* ]] || false } # ─── push ────────────────────────────────────────────────────────────── @@ -73,7 +73,7 @@ load test_helper run "$SECRETS_BIN" push testproj [ "$status" -eq 1 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } @test "push errors with missing key" { @@ -83,7 +83,7 @@ load test_helper run "$SECRETS_BIN" push testproj [ "$status" -eq 1 ] - [[ "$output" == *"Key file"* ]] + [[ "$output" == *"Key file"* ]] || false } @test "push derives project name from dirname" { @@ -140,7 +140,7 @@ load test_helper run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } # ─── pull ────────────────────────────────────────────────────────────── @@ -168,7 +168,7 @@ load test_helper run "$SECRETS_BIN" pull nonexistent [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } @test "pull errors with missing key" { @@ -183,7 +183,7 @@ load test_helper run "$SECRETS_BIN" pull testproj [ "$status" -eq 1 ] - [[ "$output" == *"Key file"* ]] + [[ "$output" == *"Key file"* ]] || false } @test "pull overwrites existing files" { @@ -217,7 +217,7 @@ load test_helper run "$SECRETS_BIN" pull testproj [ "$status" -eq 0 ] [ -x "$SECRETS_DIR/.git/hooks/pre-commit" ] - [[ "$output" == *"Reinstalled"* ]] + [[ "$output" == *"Reinstalled"* ]] || false } # ─── list ────────────────────────────────────────────────────────────── @@ -231,8 +231,8 @@ load test_helper run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"projA"* ]] - [[ "$output" == *"projB"* ]] + [[ "$output" == *"projA"* ]] || false + [[ "$output" == *"projB"* ]] || false } @test "list shows empty message" { @@ -240,7 +240,7 @@ load test_helper run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"No projects"* ]] + [[ "$output" == *"No projects"* ]] || false } # ─── rm ──────────────────────────────────────────────────────────────── @@ -261,7 +261,7 @@ load test_helper run "$SECRETS_BIN" rm nonexistent [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } # ─── pre-commit hook ────────────────────────────────────────────────── @@ -275,7 +275,7 @@ load test_helper run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } @test "pre-commit allows .age files" { @@ -304,7 +304,7 @@ load test_helper run "$SECRETS_BIN" clear [ "$status" -eq 0 ] - [[ "$output" == *"Cleared 3"* ]] + [[ "$output" == *"Cleared 3"* ]] || false # Files should be gone [ ! -f "$WORK_DIR/testproj/.env" ] @@ -318,7 +318,7 @@ load test_helper run "$SECRETS_BIN" clear [ "$status" -eq 0 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } @test "clear does not remove non-secret files" { @@ -347,7 +347,7 @@ load test_helper run "$SECRETS_BIN" clear --workspaces [ "$status" -eq 0 ] - [[ "$output" == *"Cleared"* ]] + [[ "$output" == *"Cleared"* ]] || false # All should be gone [ ! -f "$mono/.env" ] @@ -368,7 +368,7 @@ load test_helper # Run a command that reads the secret run "$SECRETS_BIN" run cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false # After run completes, plaintext files should be cleared [ ! -f "$WORK_DIR/testproj/.env" ] @@ -395,7 +395,7 @@ load test_helper @test "run errors with no command" { run "$SECRETS_BIN" run [ "$status" -eq 1 ] - [[ "$output" == *"Usage"* ]] + [[ "$output" == *"Usage"* ]] || false } @test "run passes arguments through to command" { @@ -407,7 +407,7 @@ load test_helper # Run with multiple args run "$SECRETS_BIN" run ls -la .env [ "$status" -eq 0 ] - [[ "$output" == *".env"* ]] + [[ "$output" == *".env"* ]] || false } @test "run supports -- separator" { @@ -418,7 +418,7 @@ load test_helper run "$SECRETS_BIN" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false } # ─── workspaces ──────────────────────────────────────────────────────── @@ -491,7 +491,7 @@ PKGJSON run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No package.json"* ]] + [[ "$output" == *"No package.json"* ]] || false } @test "push --workspaces errors without workspaces field" { @@ -502,7 +502,7 @@ PKGJSON run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No workspaces"* ]] + [[ "$output" == *"No workspaces"* ]] || false } @test "push --workspaces errors when no env files anywhere" { @@ -517,7 +517,7 @@ EOF run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } # ─── EGB-281: multi-store resolution ────────────────────────────────── @@ -529,8 +529,8 @@ EOF cd subdir run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "which uses .secrets-store file in cwd" { @@ -539,10 +539,10 @@ EOF create_bound_project_dir myapp "~/.secrets-work" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false # Source line must include both the rule name AND the resolved file path, # not the empty parens (".secrets-store file ()") that v0.1.0.0 shipped. - [[ "$output" == *".secrets-store file ("*"$WORK_DIR/myapp/.secrets-store)"* ]] + [[ "$output" == *".secrets-store file ("*"$WORK_DIR/myapp/.secrets-store)"* ]] || false } @test "--store flag overrides .secrets-store file and SECRETS_DIR env" { @@ -550,8 +550,8 @@ EOF create_bound_project_dir myapp "~/.secrets-from-file" run "$SECRETS_BIN" --store "$HOME/.secrets-from-flag" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-from-flag"* ]] - [[ "$output" == *"--store flag"* ]] + [[ "$output" == *"$HOME/.secrets-from-flag"* ]] || false + [[ "$output" == *"--store flag"* ]] || false } @test "which walks up to find .secrets-store in ancestor" { @@ -562,7 +562,7 @@ EOF cd "$WORK_DIR/repo/sub/deep" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "which walk-up stops at HOME boundary, does not read \$HOME/.secrets-store" { @@ -572,9 +572,9 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" != *"should-not-be-used"* ]] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" != *"should-not-be-used"* ]] || false + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "which from outside HOME falls through to default" { @@ -582,8 +582,8 @@ EOF cd /tmp run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "empty .secrets-store falls through to next rule" { @@ -593,7 +593,7 @@ EOF : > .secrets-store run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"source: default"* ]] || false } @test "comment-only .secrets-store falls through" { @@ -603,7 +603,7 @@ EOF printf '# this is a comment\n \n# another\n' > .secrets-store run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"source: default"* ]] || false } @test "bare name 'work' resolves to ~/.secrets-work" { @@ -614,7 +614,7 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "~/-prefix in .secrets-store expands to HOME" { @@ -625,7 +625,7 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-x"* ]] + [[ "$output" == *"$HOME/.secrets-x"* ]] || false } @test ".secrets-store with command injection content does not execute" { @@ -671,7 +671,7 @@ EOF run "$SECRETS_BIN" --store "$SECRETS_DIR" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false } @test "uninitialized store referenced by .secrets-store gives directed error" { @@ -683,8 +683,8 @@ EOF run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone"* ]] - [[ "$output" == *"--store"* ]] + [[ "$output" == *"git clone"* ]] || false + [[ "$output" == *"--store"* ]] || false } @test "push -w ignores per-workspace .secrets-store, uses monorepo root binding" { @@ -727,7 +727,7 @@ PKG create_project_dir myapp run "$SECRETS_BIN" --store "$HOME/.secrets-work" push myapp [ "$status" -eq 0 ] - [[ "$output" == *"Store: $HOME/.secrets-work"* ]] + [[ "$output" == *"Store: $HOME/.secrets-work"* ]] || false } # ─── EGB-281: gap-filler tests (auto-decided during /ship coverage audit) ─ @@ -735,7 +735,7 @@ PKG @test "--store with missing argument errors out" { run "$SECRETS_BIN" --store [ "$status" -eq 1 ] - [[ "$output" == *"--store requires"* ]] + [[ "$output" == *"--store requires"* ]] || false } @test "--store=value (equals form) is accepted" { @@ -744,7 +744,7 @@ PKG cd "$HOME" run "$SECRETS_BIN" --store="$HOME/.secrets-equals" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-equals"* ]] + [[ "$output" == *"$HOME/.secrets-equals"* ]] || false } @test "where and status are aliases of which" { @@ -754,11 +754,11 @@ PKG cd subdir run "$SECRETS_BIN" where [ "$status" -eq 0 ] - [[ "$output" == *"source:"* ]] + [[ "$output" == *"source:"* ]] || false run "$SECRETS_BIN" status [ "$status" -eq 0 ] - [[ "$output" == *"source:"* ]] + [[ "$output" == *"source:"* ]] || false } @test "--store default sugar resolves to ~/.secrets" { @@ -766,7 +766,7 @@ PKG cd "$HOME" run "$SECRETS_BIN" --store default which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false } @test "missing key.txt in non-default store gives directed error" { @@ -783,8 +783,8 @@ PKG run "$SECRETS_BIN" push myapp [ "$status" -eq 1 ] - [[ "$output" == *"key.txt"* ]] - [[ "$output" == *"teammate"* ]] + [[ "$output" == *"key.txt"* ]] || false + [[ "$output" == *"teammate"* ]] || false } @test "CRLF line endings in .secrets-store are tolerated" { @@ -795,7 +795,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-crlf"* ]] + [[ "$output" == *"$HOME/.secrets-crlf"* ]] || false } @test "list hints at 'secrets which' when non-default store is active" { @@ -805,7 +805,7 @@ PKG run "$SECRETS_BIN" --store "$HOME/.secrets-x" list [ "$status" -eq 0 ] - [[ "$output" == *"secrets which"* ]] + [[ "$output" == *"secrets which"* ]] || false } # ─── EGB-281: adversarial-review regression tests (F1-F5) ───────────── @@ -825,7 +825,7 @@ PKG # Run a command, then verify .env is cleared by the EXIT trap run "$SECRETS_BIN" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"should-not-leak"* ]] + [[ "$output" == *"should-not-leak"* ]] || false # CRITICAL: the trap must have cleaned up — .env must NOT exist on disk. # If F1 regressed (string-interpolated trap), the file would still be here. [ ! -f "$QUOTED_DIR/.env" ] @@ -842,27 +842,27 @@ PKG run "$SECRETS_BIN" which [ "$status" -eq 0 ] # The symlink should be ignored, falling through to default - [[ "$output" != *"/etc/passwd"* ]] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" != *"/etc/passwd"* ]] || false + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "F3: --store rejects flag-shaped value" { run "$SECRETS_BIN" --store --workspaces which [ "$status" -eq 1 ] - [[ "$output" == *"looks like a flag"* ]] + [[ "$output" == *"looks like a flag"* ]] || false } @test "F3: --store rejects literal --" { run "$SECRETS_BIN" --store -- which [ "$status" -eq 1 ] - [[ "$output" == *"looks like a flag"* ]] + [[ "$output" == *"looks like a flag"* ]] || false } @test "F4: --store= empty value is rejected" { run "$SECRETS_BIN" --store= which [ "$status" -eq 1 ] - [[ "$output" == *"requires a value"* ]] + [[ "$output" == *"requires a value"* ]] || false } @test "F5: HOME unset gives directed error" { @@ -872,7 +872,7 @@ PKG run "$SECRETS_BIN" which export HOME="$SAVED_HOME" # restore before assertions in case bats relies on it [ "$status" -ne 0 ] - [[ "$output" == *"HOME"* ]] + [[ "$output" == *"HOME"* ]] || false } # ─── EGB-282: optional remote URL in .secrets-store ────────────────── @@ -886,7 +886,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "EGB-282: .secrets-store with URL parses both tokens" { @@ -898,9 +898,9 @@ PKG # that includes the actual URL (not the placeholder). run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/work-secrets.git $HOME/.secrets-work"* ]] + [[ "$output" == *"git clone git@github.com:acme/work-secrets.git $HOME/.secrets-work"* ]] || false # Placeholder must NOT appear when a real URL was supplied - [[ "$output" != *""* ]] + [[ "$output" != *""* ]] || false } @test "EGB-282: missing-store error still works without URL (placeholder)" { @@ -911,7 +911,7 @@ PKG run "$SECRETS_BIN" pull [ "$status" -eq 1 ] # No URL given — placeholder is the right behavior. - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282: https URL is preserved literally" { @@ -921,7 +921,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"https://github.com/acme/work-secrets.git"* ]] + [[ "$output" == *"https://github.com/acme/work-secrets.git"* ]] || false } @test "EGB-282: ~/-prefixed path with URL works" { @@ -931,7 +931,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/x.git $HOME/.secrets-x"* ]] + [[ "$output" == *"git clone git@github.com:acme/x.git $HOME/.secrets-x"* ]] || false } @test "EGB-282: comments before URL line are still skipped" { @@ -941,7 +941,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/work-secrets.git"* ]] + [[ "$output" == *"git clone git@github.com:acme/work-secrets.git"* ]] || false } # ─── EGB-282 adversarial regressions: URL injection prevention ──────── @@ -958,11 +958,11 @@ PKG run "$SECRETS_BIN" pull [ "$status" -eq 1 ] # Must use the placeholder, NOT the attacker URL - [[ "$output" == *""* ]] - [[ "$output" != *"rm -rf"* ]] + [[ "$output" == *""* ]] || false + [[ "$output" != *"rm -rf"* ]] || false # And must have warned the user that something was dropped - [[ "$output" == *"WARNING"* ]] - [[ "$output" == *"unsafe"* ]] + [[ "$output" == *"WARNING"* ]] || false + [[ "$output" == *"unsafe"* ]] || false } @test "EGB-282 SECURITY: URL with backticks is dropped" { @@ -972,7 +972,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: URL with command substitution \$() is dropped" { @@ -982,7 +982,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: URL with ANSI escape is dropped (terminal-spoof prevention)" { @@ -993,7 +993,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: multi-token URL ('work url1 url2') is dropped" { @@ -1005,7 +1005,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: glob char in URL is dropped (no expansion either way)" { @@ -1018,7 +1018,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282: spec parsing is glob-safe (work * does NOT expand)" { @@ -1032,7 +1032,7 @@ PKG [ "$status" -eq 0 ] # Spec is the literal "work" (resolves to ~/.secrets-work). The "*" gets # rejected as unsafe URL and dropped. Resolution works; no globbing. - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "EGB-282: URL with - + _ : / @ . is preserved (positive test)" { @@ -1043,7 +1043,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git+ssh://user@host:2222/path/to-repo_v2.git"* ]] + [[ "$output" == *"git+ssh://user@host:2222/path/to-repo_v2.git"* ]] || false } # ─── EGB-531: gradle.properties external file support ────────────────── @@ -1069,9 +1069,9 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"gradle-properties"* ]] - [[ "$output" == *"~/.gradle/gradle.properties"* ]] - [[ "$output" == *"beaconClerkPkTest"* ]] + [[ "$output" == *"gradle-properties"* ]] || false + [[ "$output" == *"~/.gradle/gradle.properties"* ]] || false + [[ "$output" == *"beaconClerkPkTest"* ]] || false } @test "EGB-531: push extracts managed keys into external/ blob (no .env needed)" { @@ -1080,7 +1080,7 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"Extracted 2 key"* ]] + [[ "$output" == *"Extracted 2 key"* ]] || false run bash -c "ls $SECRETS_DIR/gproj/external/*.gradle-properties.age" [ "$status" -eq 0 ] } @@ -1091,7 +1091,7 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } @test "EGB-531: pull merges managed keys, preserves unrelated entries" { @@ -1103,7 +1103,7 @@ gradle_project() { gradle_src $'unrelated.key=keepme\norg.gradle.jvmargs=-Xmx2g\n' run "$SECRETS_BIN" pull gproj [ "$status" -eq 0 ] - [[ "$output" == *"Merged 2 key"* ]] + [[ "$output" == *"Merged 2 key"* ]] || false grep -q 'beaconClerkPkTest=pk_test_abc' "$HOME/.gradle/gradle.properties" grep -q 'beaconClerkPkLive=pk_live_xyz' "$HOME/.gradle/gradle.properties" grep -q 'unrelated.key=keepme' "$HOME/.gradle/gradle.properties" @@ -1225,7 +1225,7 @@ gradle_project() { printf 'gradle-properties ~/.gradle/custom.properties beaconClerkPkTest\n' > .secrets-files run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"gradle.properties"* ]] + [[ "$output" == *"gradle.properties"* ]] || false } @test "EGB-531: target outside HOME is refused" { @@ -1239,7 +1239,7 @@ gradle_project() { run "$SECRETS_BIN" push gproj rm -rf "$outside" [ "$status" -eq 1 ] - [[ "$output" == *"HOME"* ]] + [[ "$output" == *"HOME"* ]] || false } @test "EGB-531: symlinked target is refused" { @@ -1250,7 +1250,7 @@ gradle_project() { gradle_project gproj beaconClerkPkTest run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"symlink"* ]] + [[ "$output" == *"symlink"* ]] || false } @test "EGB-531: unknown type in manifest warns and skips" { @@ -1259,7 +1259,7 @@ gradle_project() { printf 'gradle-props ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files echo "X=1" > .env run "$SECRETS_BIN" push gproj - [[ "$output" == *"unknown type"* ]] + [[ "$output" == *"unknown type"* ]] || false [ ! -d "$SECRETS_DIR/gproj/external" ] } @@ -1269,7 +1269,7 @@ gradle_project() { printf 'gradle-properties\n' > .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"WARNING"* ]] + [[ "$output" == *"WARNING"* ]] || false } @test "EGB-531: manifest path with command-substitution chars is rejected" { @@ -1280,7 +1280,7 @@ gradle_project() { printf 'gradle-properties ~/.gradle/gradle.properties$(touch %s) beaconClerkPkTest\n' "$pwn" > .secrets-files run "$SECRETS_BIN" which [ ! -f "$pwn" ] - [[ "$output" == *"WARNING"* ]] + [[ "$output" == *"WARNING"* ]] || false } @test "EGB-531: symlinked .secrets-files is ignored" { @@ -1290,7 +1290,7 @@ gradle_project() { ln -s "$HOME/realmanifest" .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" != *"beaconClerkPkTest"* ]] + [[ "$output" != *"beaconClerkPkTest"* ]] || false } @test "EGB-531: rekey re-encrypts the external blob (still decryptable after)" { @@ -1329,7 +1329,7 @@ gradle_project() { "$SECRETS_BIN" push gproj >/dev/null 2>&1 run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"external"* ]] + [[ "$output" == *"external"* ]] || false } @test "EGB-531: no .secrets-files behaves exactly as before (backward compat)" { @@ -1347,7 +1347,7 @@ gradle_project() { git add -f gradle.properties run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } # ── EGB-531: coverage for warning/error branches, workspaces, multi-entry ── @@ -1395,9 +1395,9 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"beaconClerkPkLive"* ]] - [[ "$output" == *"not found"* ]] - [[ "$output" == *"Extracted 1 key"* ]] + [[ "$output" == *"beaconClerkPkLive"* ]] || false + [[ "$output" == *"not found"* ]] || false + [[ "$output" == *"Extracted 1 key"* ]] || false } @test "EGB-531: pull warns when manifest entry has no blob in store" { @@ -1408,7 +1408,7 @@ gradle_project() { cd "$WORK_DIR/gproj" run "$SECRETS_BIN" pull gproj [ "$status" -eq 0 ] - [[ "$output" == *"no encrypted data exists"* ]] + [[ "$output" == *"no encrypted data exists"* ]] || false } @test "EGB-531: multi-entry manifest syncs each target" { @@ -1436,8 +1436,8 @@ gradle_project() { printf 'gradle-properties ~/.gradle/gradle.properties bad=key\n' > .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"WARNING"* ]] - [[ "$output" != *"bad=key"* ]] + [[ "$output" == *"WARNING"* ]] || false + [[ "$output" != *"bad=key"* ]] || false } @test "EGB-531: symlinked parent dir of target is refused" { @@ -1448,7 +1448,7 @@ gradle_project() { gradle_project gproj beaconClerkPkTest run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"symlink"* ]] + [[ "$output" == *"symlink"* ]] || false } @test "EGB-531: push skips a multi-line (continuation) managed value with a warning" { @@ -1459,8 +1459,8 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"multi-line"* ]] - [[ "$output" == *"Extracted 1 key"* ]] + [[ "$output" == *"multi-line"* ]] || false + [[ "$output" == *"Extracted 1 key"* ]] || false } @test "EGB-531: push skips comment and continuation lines in source" { @@ -1491,7 +1491,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone"* ]] + [[ "$output" == *"git clone"* ]] || false # Must not leave a half-initialized store behind [ ! -d "$SECRETS_DIR/.git" ] # Key untouched @@ -1505,12 +1505,12 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"Restored store .gitignore"* ]] + [[ "$output" == *"Restored store .gitignore"* ]] || false [ -f "$SECRETS_DIR/.gitignore" ] grep -q "key.txt" "$SECRETS_DIR/.gitignore" # key.txt must never be tracked (push does `git add -A` in the store) run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "pull restores missing store .gitignore" { @@ -1535,7 +1535,7 @@ gradle_project() { [ "$status" -eq 0 ] [ -f "$SECRETS_DIR/.gitignore" ] run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "rekey re-encrypts dotenv blobs (round-trip survives key rotation)" { @@ -1563,7 +1563,7 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"Reinstalled pre-commit hook"* ]] + [[ "$output" == *"Reinstalled pre-commit hook"* ]] || false [ -x "$SECRETS_DIR/.git/hooks/pre-commit" ] } @@ -1584,8 +1584,8 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" != *"Restored store .gitignore"* ]] - [[ "$output" != *"Reinstalled pre-commit hook"* ]] + [[ "$output" != *"Restored store .gitignore"* ]] || false + [[ "$output" != *"Reinstalled pre-commit hook"* ]] || false } @test "restored store .gitignore carries the full block/allow globs" { @@ -1617,7 +1617,7 @@ gradle_project() { [ "$status" -eq 0 ] [ -f "$SECRETS_DIR/.gitignore" ] run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "push untracks a previously committed key.txt with a warning" { @@ -1629,9 +1629,9 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"key.txt was tracked"* ]] + [[ "$output" == *"key.txt was tracked"* ]] || false run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "push rewrites a store .gitignore that is missing the key.txt line" { @@ -1643,7 +1643,7 @@ gradle_project() { [ "$status" -eq 0 ] grep -qx 'key.txt' "$SECRETS_DIR/.gitignore" run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "init guard renders the real clone URL when .secrets-store carries a remote" { @@ -1655,7 +1655,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] + [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] || false } # ─── EGB-652: `file` external type (whole-file sync, e.g. Android keystore) ── @@ -1680,8 +1680,8 @@ file_project() { file_project fproj run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"file"* ]] - [[ "$output" == *"~/keystores/upload.keystore"* ]] + [[ "$output" == *"file"* ]] || false + [[ "$output" == *"~/keystores/upload.keystore"* ]] || false } @test "EGB-652: push encrypts a file-type entry into external/ blob" { @@ -1690,7 +1690,7 @@ file_project() { file_project fproj run "$SECRETS_BIN" push fproj [ "$status" -eq 0 ] - [[ "$output" == *"Encrypted file"* ]] + [[ "$output" == *"Encrypted file"* ]] || false run bash -c "ls $SECRETS_DIR/fproj/external/*.file.age" [ "$status" -eq 0 ] } @@ -1704,7 +1704,7 @@ file_project() { rm -rf "$HOME/keystores" run "$SECRETS_BIN" pull fproj [ "$status" -eq 0 ] - [[ "$output" == *"Restored file"* ]] + [[ "$output" == *"Restored file"* ]] || false cmp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/reference" mode=$(stat -f '%Lp' "$HOME/keystores/upload.keystore" 2>/dev/null || stat -c '%a' "$HOME/keystores/upload.keystore") [ "$mode" = "600" ] @@ -1730,10 +1730,10 @@ file_project() { cd "$dir" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"take no keys"* ]] + [[ "$output" == *"take no keys"* ]] || false # The rejected entry must not be listed as parsed (header only prints # when at least one entry parses). - [[ "$output" != *"external files ("* ]] + [[ "$output" != *"external files ("* ]] || false } @test "EGB-652: file target outside HOME is refused on push" { @@ -1743,7 +1743,7 @@ file_project() { cd "$dir" run "$SECRETS_BIN" push fout [ "$status" -ne 0 ] - [[ "$output" == *"inside \$HOME"* ]] || [[ "$output" == *"Refusing"* ]] + [[ "$output" == *"inside \$HOME"* ]] || [[ "$output" == *"Refusing"* ]] || false } @test "EGB-652: gradle-properties entries still work alongside a file entry" { @@ -1755,6 +1755,6 @@ file_project() { cd "$dir" run "$SECRETS_BIN" push fmix [ "$status" -eq 0 ] - [[ "$output" == *"Extracted 1 key"* ]] - [[ "$output" == *"Encrypted file"* ]] + [[ "$output" == *"Extracted 1 key"* ]] || false + [[ "$output" == *"Encrypted file"* ]] || false } From 18018dbd3b4e03e46e8af67b8ddc6ee373a4ddc6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:24:35 -0700 Subject: [PATCH 02/70] =?UTF-8?q?feat:=20.secrets.json=20manifest=20core?= =?UTF-8?q?=20=E2=80=94=20add=20command,=20rails,=20canonical=20form=20(EG?= =?UTF-8?q?B-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - secrets add : explicit manifest writer, idempotent, atomic write - _validate_dotenv_rel_path: project-relative confinement rail (no .. / absolute / shell metas; @ allowed for npm-scoped workspace dirs) - _check_manifest_file: refuses symlinks, malformed JSON (jq error with file named), unsupported schema versions (directed upgrade error) - canonical serialization: jq --sort-keys + sorted/deduped dotenv — add order produces byte-identical manifests - which: validates + summarizes the manifest (doubles as linter) - jq required only when a manifest exists/is written --- secrets | 147 +++++++++++++++++++++++++++++++++++++++++++++ test/manifest.bats | 125 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 test/manifest.bats diff --git a/secrets b/secrets index 508e5de..99fdae4 100755 --- a/secrets +++ b/secrets @@ -712,6 +712,129 @@ pull_external_files() { done < <(_parse_secrets_files_manifest "$manifest") } +# ─── Manifest (.secrets.json) — EGB-677 store format v2, stage 1 ────── +# +# A committed, project-root JSON manifest declaring everything the project +# syncs. Stage 1 is ADDITIVE: the manifest is read alongside the existing +# dotenv globs and `.secrets-files`; the v1 store layout is unchanged. +# +# { +# "version": 2, +# "options": { "autoAdd": true }, +# "dotenv": [".env", "packages/web/.env.development"], +# "external": [ +# { "type": "properties", "path": "~/.gradle/gradle.properties", +# "keys": ["beaconClerkPk"] }, +# { "type": "file", "path": "~/keystores/beacon-upload.keystore" } +# ] +# } +# +# Security posture mirrors `.secrets-store`/`.secrets-files`: symlinked +# manifests are refused, dotenv paths are confined to the project root +# (no `..`, no absolute paths, conservative charset — `@` allowed for +# npm-scoped workspace dirs), and every jq-extracted string is +# re-validated before any filesystem use. jq is required only when a +# manifest exists or is being written — manifest-less projects keep +# working with zero new dependencies. + +SECRETS_JSON_NAME=".secrets.json" +MANIFEST_VERSION=2 + +# Validate a project-relative dotenv path from the manifest (or `secrets +# add`). The path is attacker-influenced in team repos (committed file), +# so this is a security rail, not just hygiene. Prints an error and +# returns 1 when unsafe. +_validate_dotenv_rel_path() { + local p="$1" + if [ -z "$p" ]; then + echo "ERROR: $SECRETS_JSON_NAME: empty dotenv path." >&2 + return 1 + fi + case "$p" in + /*) + echo "ERROR: $SECRETS_JSON_NAME: '$p' is absolute — dotenv paths must be project-relative." >&2 + return 1 + ;; + *..*) + echo "ERROR: $SECRETS_JSON_NAME: '$p' contains '..' — dotenv paths must be project-relative (no traversal)." >&2 + return 1 + ;; + *[!A-Za-z0-9@/._-]*) + echo "ERROR: $SECRETS_JSON_NAME: unsafe characters in '$p'. Paths may contain letters, digits, @ / . _ - only." >&2 + return 1 + ;; + esac + return 0 +} + +# Validate the manifest file itself (existence assumed checked by caller): +# refuse symlinks, malformed JSON, and unsupported schema versions — each +# with a directed error. $1 = manifest path. +_check_manifest_file() { + local manifest="$1" + if [ -L "$manifest" ]; then + die "$manifest is a symlink. Refusing to read it. + A committed symlink could point the manifest at attacker-chosen content. + Replace it with a regular file." + fi + check_cmd jq + local jq_err + if ! jq_err=$(jq -e . "$manifest" 2>&1 >/dev/null); then + die "$manifest: invalid JSON. + $jq_err + Fix the syntax (or delete the file and re-run 'secrets add' / 'secrets push')." + fi + local ver + ver=$(jq -r '.version // "missing"' "$manifest") + if [ "$ver" != "$MANIFEST_VERSION" ]; then + die "$manifest: manifest version $ver is not supported. + This client supports version $MANIFEST_VERSION. + If the manifest was written by a newer secrets, upgrade this machine: + git -C $SCRIPT_DIR pull" + fi +} + +# Canonically (re)write the manifest: sorted keys, sorted+deduped dotenv, +# atomic tmp+mv in the project dir. stdin = the new JSON document. +_write_manifest_canonical() { + local manifest="$1" + local dir; dir=$(dirname "$manifest") + local tmp; tmp=$(mktemp "$dir/.secrets-json.XXXXXX") || return 1 + if ! jq --sort-keys '.dotenv |= ((. // []) | unique | sort)' > "$tmp"; then + rm -f "$tmp" + return 1 + fi + mv "$tmp" "$manifest" +} + +# `secrets add ` — the explicit manifest writer. Creates +# .secrets.json on first use; validates and dedupes thereafter. +cmd_add() { + local p="${1:-}" + [ -n "$p" ] || die "Usage: secrets add " + # Normalize a leading ./ + p="${p#./}" + _validate_dotenv_rel_path "$p" || exit 1 + if [ ! -f "$PWD/$p" ]; then + die "'$p' not found in $PWD. + Create the file first, then re-run: secrets add $p" + fi + check_cmd jq + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + jq --arg p "$p" '.dotenv = ((.dotenv // []) + [$p])' "$manifest" \ + | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" + else + jq -n --arg p "$p" '{version: '"$MANIFEST_VERSION"', dotenv: [$p]}' \ + | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" + fi + info "Added '$p' to $SECRETS_JSON_NAME" + info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." +} + +# ─── End manifest ────────────────────────────────────────────────────── + # Read package.json workspaces and expand globs to actual directories. # Prints one workspace path per line (relative to the monorepo root). get_workspaces() { @@ -1397,6 +1520,29 @@ cmd_which() { echo "store: $SECRETS_DIR" echo "source: $STORE_SOURCE" + # v2 manifest (.secrets.json): validate and summarize. Validation here + # is deliberately fatal (symlink / malformed / unsupported version) so + # `secrets which` doubles as the manifest linter. + local json_manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$json_manifest" ]; then + _check_manifest_file "$json_manifest" + echo "manifest ($SECRETS_JSON_NAME at $json_manifest):" + local entry + while IFS= read -r entry; do + [ -n "$entry" ] || continue + if _validate_dotenv_rel_path "$entry"; then + echo " dotenv $entry" + else + echo " dotenv $entry [UNSAFE — will be refused]" + fi + done < <(jq -r '.dotenv // [] | .[]' "$json_manifest") + local etype epath ekeys + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + echo " $etype $epath $ekeys" + done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$json_manifest") + fi + # Read back any external-file manifest in cwd (validates the format and # gives the user a way to confirm it parsed, since there's no add-file # command). Skips symlinked manifests. @@ -1569,6 +1715,7 @@ case "${1:-help}" in shift cmd_run "$@" ;; + add) cmd_add "${2:-}" ;; list) cmd_list ;; rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; diff --git a/test/manifest.bats b/test/manifest.bats new file mode 100644 index 0000000..eb3e7d2 --- /dev/null +++ b/test/manifest.bats @@ -0,0 +1,125 @@ +#!/usr/bin/env bats +# EGB-677 stage 1: .secrets.json manifest — parse, rails, add, generators. + +load test_helper + +# ─── A: manifest core — secrets add + rails + canonical form ────────── + +@test "add creates .secrets.json with version 2 and the dotenv entry" { + create_project_dir addproj + run "$SECRETS_BIN" add .env + [ "$status" -eq 0 ] + [ -f ".secrets.json" ] + run jq -r '.version' .secrets.json + [ "$output" = "2" ] + run jq -r '.dotenv[0]' .secrets.json + [ "$output" = ".env" ] +} + +@test "add is idempotent — no duplicate entries" { + create_project_dir addproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" add .env + [ "$status" -eq 0 ] + run jq -r '.dotenv | length' .secrets.json + [ "$output" = "1" ] +} + +@test "add accepts nested workspace paths" { + create_project_dir addproj + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + run "$SECRETS_BIN" add packages/web/.env.development + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/web/.env.development") != null' .secrets.json + [ "$output" = "true" ] +} + +@test "add accepts npm-scoped workspace paths (@)" { + create_project_dir addproj + mkdir -p "packages/@acme/web" + echo "K=v" > "packages/@acme/web/.env" + run "$SECRETS_BIN" add "packages/@acme/web/.env" + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/@acme/web/.env") != null' .secrets.json + [ "$output" = "true" ] +} + +@test "add rejects path traversal (..)" { + create_project_dir addproj + run "$SECRETS_BIN" add ../escape/.env + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false + [ ! -f ".secrets.json" ] +} + +@test "add rejects absolute paths" { + create_project_dir addproj + run "$SECRETS_BIN" add /etc/passwd + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false + [ ! -f ".secrets.json" ] +} + +@test "add rejects shell metacharacters in path" { + create_project_dir addproj + run "$SECRETS_BIN" add '.env;rm -rf ~' + [ "$status" -eq 1 ] + [ ! -f ".secrets.json" ] +} + +@test "add requires the file to exist" { + create_project_dir addproj + run "$SECRETS_BIN" add .env.missing + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] || false +} + +@test "manifest serialization is canonical — order of adds does not matter" { + create_project_dir addproj + echo "A=1" > .env.alpha + echo "B=2" > .env.beta + "$SECRETS_BIN" add .env.alpha >/dev/null + "$SECRETS_BIN" add .env.beta >/dev/null + cp .secrets.json "$TEST_TMPDIR/order1.json" + rm .secrets.json + "$SECRETS_BIN" add .env.beta >/dev/null + "$SECRETS_BIN" add .env.alpha >/dev/null + cmp -s .secrets.json "$TEST_TMPDIR/order1.json" +} + +@test "which shows manifest summary when .secrets.json is present" { + create_project_dir addproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *".env"* ]] || false +} + +@test "malformed .secrets.json dies with a directed error naming the file" { + create_project_dir addproj + echo '{ not json' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *"invalid"* ]] || false +} + +@test "unsupported manifest version dies with a directed upgrade error" { + create_project_dir addproj + echo '{"version": 99, "dotenv": [".env"]}' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *"version 99"* ]] || false + [[ "$output" == *"supports"* ]] || false +} + +@test "symlinked .secrets.json is refused" { + create_project_dir addproj + echo '{"version":2,"dotenv":[".env"]}' > "$TEST_TMPDIR/real-manifest.json" + ln -s "$TEST_TMPDIR/real-manifest.json" .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *"symlink"* ]] || false +} From 884da0965cfebc99bd203c52ab4a268b1747a0fa Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:32:17 -0700 Subject: [PATCH 03/70] =?UTF-8?q?feat:=20manifest-aware=20push=20=E2=80=94?= =?UTF-8?q?=20generator=20auto-add,=20autoAdd=20toggle,=20--frozen/--dry-r?= =?UTF-8?q?un=20(EGB-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - push syncs FROM the manifest; v1 store layout unchanged (nested entries land at /.age, same shape -w always used) - discovery (root globs + quiet package.json workspace re-scan when a manifest exists) feeds the manifest as a generator; new files auto-add with ==> notice + undo guidance - options.autoAdd committed toggle (default ON when absent); explicit false warns on undeclared files instead of enrolling them - push --frozen: declared-only for one invocation; push --dry-run: reports would-add/would-sync, touches nothing - bootstrap ordering: manifest written only after >=1 blob encrypts - declared-but-missing warns and continues; unsafe manifest path dies - jq // falsy gotcha: explicit autoAdd:false compared directly --- secrets | 147 ++++++++++++++++++++++++++++++++++++++++++++- test/manifest.bats | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 3 deletions(-) diff --git a/secrets b/secrets index 99fdae4..e1d4f2c 100755 --- a/secrets +++ b/secrets @@ -833,6 +833,27 @@ cmd_add() { info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." } +# Quietly emit "ws-dir/basename" for every env file in a package.json +# workspace under . Emits nothing (and never dies) when is +# not a workspace monorepo or jq is unavailable — plain `push` calls this +# speculatively so a new workspace's env files keep getting discovered +# after the one-time --workspaces generator run (EGB-677 E13). +_maybe_workspace_env_files() { + local root="$1" + [ -f "$root/package.json" ] || return 0 + command -v jq >/dev/null 2>&1 || return 0 + jq -e '.workspaces' "$root/package.json" >/dev/null 2>&1 || return 0 + local ws f + while IFS= read -r ws; do + [ -n "$ws" ] || continue + if collect_env_files "$root/$ws"; then + for f in "${COLLECTED_FILES[@]}"; do + printf '%s/%s\n' "$ws" "$(basename "$f")" + done + fi + done < <(get_workspaces "$root") +} + # ─── End manifest ────────────────────────────────────────────────────── # Read package.json workspaces and expand globs to actual directories. @@ -1033,7 +1054,22 @@ commit_and_push_secrets() { fi } +# Manifest-aware push (EGB-677 stage 1). Discovery (root globs + a quiet +# package.json workspace re-scan when a manifest exists) feeds the +# manifest as a GENERATOR; the sync itself runs FROM the manifest. The +# v1 store layout is unchanged: root files land at /.age, +# nested entries at /.age (same shape -w always used). cmd_push() { + local frozen=false dry_run=false explicit_project="" + while [ $# -gt 0 ]; do + case "$1" in + --frozen) frozen=true; shift ;; + --dry-run) dry_run=true; shift ;; + -*) die "Unknown push flag: $1. Usage: secrets push [--frozen] [--dry-run] [project]" ;; + *) explicit_project="$1"; shift ;; + esac + done + check_cmd age check_cmd git resolve_store @@ -1041,20 +1077,124 @@ cmd_push() { check_key local project - project=$(derive_project_name "${1:-}") + project=$(derive_project_name "$explicit_project") info "Pushing secrets for project: $project" echo_store_if_non_default local pubkey pubkey=$(get_pubkey) + # ── Manifest read (validated; absence = bootstrap) ── + local manifest="$PWD/$SECRETS_JSON_NAME" + local have_manifest=false auto_add=true declared="" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + have_manifest=true + declared=$(jq -r '.dotenv // [] | .[]' "$manifest") + local d + while IFS= read -r d; do + [ -n "$d" ] || continue + _validate_dotenv_rel_path "$d" \ + || die "Refusing unsafe dotenv path in $SECRETS_JSON_NAME (paths must be project-relative): $d" + done <<< "$declared" + # NB: jq's // treats false as empty, so `.options.autoAdd // true` + # would silently flip an explicit false back to true. Compare directly. + auto_add=$(jq -r '.options.autoAdd | if . == false then "false" else "true" end' "$manifest") + fi + [ "$frozen" = true ] && auto_add=false + + # ── Discovery: root globs + workspace re-scan (manifest projects) ── + local discovered="" f + if collect_env_files "$PWD"; then + for f in "${COLLECTED_FILES[@]}"; do + discovered="$discovered$(basename "$f")"$'\n' + done + fi + if [ "$have_manifest" = true ]; then + discovered="$discovered$(_maybe_workspace_env_files "$PWD")"$'\n' + fi + + # to_add = discovered − declared (deduped; pure bash 3.2, no assoc arrays) + local to_add="" e known + while IFS= read -r e; do + [ -n "$e" ] || continue + known=0 + while IFS= read -r d; do [ "$d" = "$e" ] && { known=1; break; }; done <<< "$declared" + [ "$known" -eq 1 ] && continue + while IFS= read -r d; do [ "$d" = "$e" ] && { known=1; break; }; done <<< "$to_add" + [ "$known" -eq 1 ] && continue + to_add="$to_add$e"$'\n' + done <<< "$discovered" + + if [ "$dry_run" = true ]; then + info "Dry run — nothing encrypted, nothing written." + if [ -n "$to_add" ]; then + echo "Would add to $SECRETS_JSON_NAME:" + while IFS= read -r e; do [ -n "$e" ] && echo " $e"; done <<< "$to_add" + else + echo "Nothing new to add to $SECRETS_JSON_NAME." + fi + if [ -n "$declared" ]; then + echo "Would sync (declared):" + while IFS= read -r e; do [ -n "$e" ] && echo " $e"; done <<< "$declared" + fi + return 0 + fi + + # ── Build the sync list ── + local sync_list="$declared" + if [ "$auto_add" = true ] || [ "$have_manifest" = false ]; then + sync_list="$declared"$'\n'"$to_add" + else + while IFS= read -r e; do + [ -n "$e" ] || continue + echo "WARNING: '$e' is not declared in $SECRETS_JSON_NAME and autoAdd is off — not synced. Run: secrets add $e" >&2 + done <<< "$to_add" + fi + + # ── Encrypt FROM the (effective) manifest ── + local count=0 rel + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if [ ! -f "$PWD/$rel" ]; then + echo "WARNING: '$rel' is declared in $SECRETS_JSON_NAME but not found in $PWD — skipping." >&2 + continue + fi + case "$rel" in + */*) mkdir -p "$SECRETS_DIR/$project/$(dirname "$rel")" ;; + *) mkdir -p "$SECRETS_DIR/$project" ;; + esac + age -r "$pubkey" -o "$SECRETS_DIR/$project/${rel}.age" "$PWD/$rel" + echo " $rel" + count=$((count + 1)) + done <<< "$sync_list" + [ "$count" -gt 0 ] && info "$project: $count file(s)" + local did=0 - if push_dir_to_project "$PWD" "$project" "$pubkey"; then did=1; fi + [ "$count" -gt 0 ] && did=1 if push_external_files "$PWD" "$project" "$pubkey"; then did=1; fi if [ "$did" -eq 0 ]; then die "No secret files (.env, .env.*, .dev.vars) or $SECRETS_FILES_NAME entries found in $PWD" fi + # ── Manifest write AFTER successful encryption (bootstrap ordering) ── + if [ "$count" -gt 0 ] && [ -n "$to_add" ] && [ "$frozen" = false ] \ + && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then + local add_json + add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') + if [ "$have_manifest" = true ]; then + jq --argjson add "$add_json" '.dotenv = ((.dotenv // []) + $add)' "$manifest" \ + | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" + else + jq -n --argjson add "$add_json" '{version: '"$MANIFEST_VERSION"', dotenv: $add}' \ + | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" + fi + while IFS= read -r e; do + [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" + done <<< "$to_add" + info "Commit the manifest so other machines pick it up. To undo an entry: edit $SECRETS_JSON_NAME (or use 'secrets push --frozen' to skip auto-add)." + fi + commit_and_push_secrets "update $project" } @@ -1694,7 +1834,8 @@ case "${1:-help}" in if [ "${2:-}" = "-w" ] || [ "${2:-}" = "--workspaces" ]; then cmd_push_workspaces else - cmd_push "${2:-}" + shift + cmd_push "$@" fi ;; pull) diff --git a/test/manifest.bats b/test/manifest.bats index eb3e7d2..9800c72 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -123,3 +123,124 @@ load test_helper [ "$status" -eq 1 ] [[ "$output" == *"symlink"* ]] || false } + +# ─── B: push from manifest — generators, autoAdd, --frozen/--dry-run ─── + +@test "push with manifest syncs nested declared file into v1 store layout" { + init_with_remote + create_project_dir nestproj + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/nestproj/packages/web/.env.development.age" ] +} + +@test "push auto-adds newly discovered root files to an existing manifest" { + init_with_remote + create_project_dir autoproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"Added"* ]] || false + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "true" ] + [ -f "$SECRETS_DIR/autoproj/.env.staging.age" ] +} + +@test "bootstrap: plain push creates the manifest from discovered files" { + init_with_remote + create_project_dir bootproj + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f ".secrets.json" ] + run jq -r '.dotenv | length' .secrets.json + [ "$output" = "2" ] +} + +@test "failed push leaves no bootstrap manifest behind" { + init_with_remote + mkdir -p "$WORK_DIR/emptyproj" + cd "$WORK_DIR/emptyproj" + run "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [ ! -f ".secrets.json" ] +} + +@test "autoAdd=false: undeclared discovered file is warned about, not added or synced" { + init_with_remote + create_project_dir noaddproj + printf '{"version":2,"options":{"autoAdd":false},"dotenv":[".env"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"not declared"* ]] || false + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "false" ] + [ -f "$SECRETS_DIR/noaddproj/.env.age" ] + [ ! -f "$SECRETS_DIR/noaddproj/.env.staging.age" ] +} + +@test "push --frozen skips auto-add even when autoAdd is on" { + init_with_remote + create_project_dir frozenproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push --frozen + [ "$status" -eq 0 ] + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "false" ] + [ ! -f "$SECRETS_DIR/frozenproj/.env.staging.age" ] + # declared entry still synced under the REAL project name + [ -f "$SECRETS_DIR/frozenproj/.env.age" ] +} + +@test "push --dry-run reports would-add entries and changes nothing" { + init_with_remote + create_project_dir dryproj + "$SECRETS_BIN" add .env >/dev/null + cp .secrets.json "$TEST_TMPDIR/manifest-before.json" + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *".env.staging"* ]] || false + cmp -s .secrets.json "$TEST_TMPDIR/manifest-before.json" + [ ! -f "$SECRETS_DIR/dryproj/.env.age" ] + # nothing committed to the store at all + [ "$(git -C "$SECRETS_DIR" rev-list --count HEAD)" -eq 1 ] +} + +@test "plain push re-scans package.json workspaces when a manifest exists" { + init_with_remote + local mono="$WORK_DIR/wsproj" + mkdir -p "$mono/packages/api" + printf '{"workspaces": ["packages/*"]}\n' > "$mono/package.json" + echo "ROOT=1" > "$mono/.env" + echo "API=1" > "$mono/packages/api/.dev.vars" + git init "$mono" >/dev/null 2>&1 + cd "$mono" + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/api/.dev.vars") != null' .secrets.json + [ "$output" = "true" ] + [ -f "$SECRETS_DIR/wsproj/packages/api/.dev.vars.age" ] +} + +@test "declared-but-missing file warns and push continues" { + init_with_remote + create_project_dir missproj + "$SECRETS_BIN" add .env >/dev/null + printf '{"version":2,"dotenv":[".env",".env.gone"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *".env.gone"* ]] || false + [ -f "$SECRETS_DIR/missproj/.env.age" ] +} + +@test "unsafe dotenv entry in a committed manifest dies on push" { + init_with_remote + create_project_dir evilproj + printf '{"version":2,"dotenv":["../escape/.env"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false +} From d7e1400487eacb9b41120a9b596450e688a41c11 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:44:02 -0700 Subject: [PATCH 04/70] feat: external entries via .secrets.json + legacy absorb + properties rail (EGB-677 stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .secrets.json external[] drives push/pull: type 'properties' (alias of gradle-properties; blob suffix stays legacy-compatible in stage 1) and type 'file'; same charset rails as the legacy parser - push absorbs uncovered .secrets-files entries into the manifest (idempotent, gradle-properties → properties) with a delete hint - pull: manifest wins entirely; a coexisting .secrets-files warns as superseded instead of being silently ignored - basename rail generalized: properties targets must end '.properties' (was exact 'gradle.properties') — rc files/gitconfig still blocked; EGB-531 wrong-basename test updated for the sanctioned change --- secrets | 206 +++++++++++++++++++++++++++++++++++++++------ test/manifest.bats | 135 +++++++++++++++++++++++++++++ test/secrets.bats | 13 ++- 3 files changed, 322 insertions(+), 32 deletions(-) diff --git a/secrets b/secrets index e1d4f2c..e02e636 100755 --- a/secrets +++ b/secrets @@ -425,9 +425,17 @@ _parse_secrets_files_manifest() { _validate_external_target_path() { local p="$1" mtype="${2:-gradle-properties}" local base; base=$(basename "$p") - if [ "$mtype" = "gradle-properties" ] && [ "$base" != "gradle.properties" ]; then - echo "ERROR: $SECRETS_FILES_NAME: target basename must be 'gradle.properties' (got '$base'). Refusing." >&2 - return 1 + if [ "$mtype" = "gradle-properties" ]; then + # EGB-677: generalized from exact 'gradle.properties' to any + # '*.properties' basename — still blocks merging key=value lines + # into ~/.bashrc / ~/.gitconfig style targets. + case "$base" in + *.properties) ;; + *) + echo "ERROR: properties target basename must end in '.properties' (got '$base'). Refusing." >&2 + return 1 + ;; + esac fi case "$p" in *..*) echo "ERROR: $SECRETS_FILES_NAME: target path may not contain '..'. Refusing." >&2; return 1 ;; esac local home_real; home_real=$(cd -P "$HOME" 2>/dev/null && pwd -P) || home_real="$HOME" @@ -575,13 +583,11 @@ merge_gradle_keys() { # there is no usable manifest. Dies on unsafe targets or all-missing keys. push_external_files() { local root="$1" project="$2" pubkey="$3" - local manifest="$root/$SECRETS_FILES_NAME" - [ -e "$manifest" ] || return 1 - if [ -L "$manifest" ]; then - echo "WARNING: $manifest is a symlink; ignoring." >&2 - return 1 - fi - [ -f "$manifest" ] || return 1 + # Entries come from .secrets.json (EGB-677) plus any legacy + # .secrets-files entries the manifest doesn't cover yet. + local entries + entries=$(_external_entries_for_push "$root") + [ -n "$entries" ] || return 1 local pushed=0 mtype mpath mkeys while IFS=$'\t' read -r mtype mpath mkeys; do @@ -631,7 +637,7 @@ push_external_files() { rm -f "$tmp" info "Extracted $found key(s) from $mpath" pushed=$((pushed + 1)) - done < <(_parse_secrets_files_manifest "$manifest") + done <<< "$entries" [ "$pushed" -gt 0 ] } @@ -641,13 +647,11 @@ push_external_files() { # missing blobs rather than aborting the whole pull. pull_external_files() { local root="$1" project="$2" - local manifest="$root/$SECRETS_FILES_NAME" - [ -e "$manifest" ] || return 0 - if [ -L "$manifest" ]; then - echo "WARNING: $manifest is a symlink; ignoring." >&2 - return 0 - fi - [ -f "$manifest" ] || return 0 + # .secrets.json wins entirely when present (EGB-677); legacy + # .secrets-files only drives manifest-less projects. + local entries + entries=$(_external_entries_for_pull "$root") + [ -n "$entries" ] || return 0 local mtype mpath mkeys while IFS=$'\t' read -r mtype mpath mkeys; do @@ -709,7 +713,7 @@ pull_external_files() { echo "WARNING: failed to merge keys into $expanded — target left unchanged." >&2 fi rm -f "$tmp" - done < <(_parse_secrets_files_manifest "$manifest") + done <<< "$entries" } # ─── Manifest (.secrets.json) — EGB-677 store format v2, stage 1 ────── @@ -833,6 +837,133 @@ cmd_add() { info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." } +# Emit "\t\t" tuples from a .secrets.json external[] +# array — the same wire format _parse_secrets_files_manifest produces, so +# push_external_files / pull_external_files consume either source +# unchanged. JSON type 'properties' maps to the legacy tuple token +# 'gradle-properties' so blob suffixes (and existing store blobs) stay +# stable in stage 1. Applies the same conservative charset checks as the +# legacy parser — jq guarantees well-formed JSON, not safe VALUES. +_json_external_entries() { + local manifest="$1" + check_cmd jq + local etype epath ekeys + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + case "$etype" in + properties|gradle-properties) + etype="gradle-properties" + if [ -z "$ekeys" ]; then + echo "WARNING: $SECRETS_JSON_NAME: properties entry '$epath' has no keys. Skipping." >&2 + continue + fi + ;; + file) + if [ -n "$ekeys" ]; then + echo "WARNING: $SECRETS_JSON_NAME: 'file' entries take no keys ('$epath' lists '$ekeys'). Skipping." >&2 + continue + fi + ;; + *) + echo "WARNING: $SECRETS_JSON_NAME: unknown external type '$etype' (supported: properties file). Skipping." >&2 + continue + ;; + esac + case "$epath" in + ''|*[!A-Za-z0-9/._~-]*|*..*) + echo "WARNING: $SECRETS_JSON_NAME: unsafe characters in external path '$epath'. Skipping." >&2 + continue + ;; + esac + case "$ekeys" in + *[!A-Za-z0-9._\ -]*) + echo "WARNING: $SECRETS_JSON_NAME: unsafe characters in key list for '$epath'. Skipping." >&2 + continue + ;; + esac + printf '%s\t%s\t%s\n' "$etype" "$epath" "$ekeys" + done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$manifest") +} + +# External tuples for PUSH: .secrets.json entries first, then legacy +# .secrets-files entries whose (type, path) the manifest doesn't cover — +# the absorb set, which cmd_push folds into the manifest after a +# successful push so the two sources converge. +_external_entries_for_push() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + local seen="" t p k + if [ -f "$json" ] && [ ! -L "$json" ]; then + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + printf '%s\t%s\t%s\n' "$t" "$p" "$k" + seen="$seen$t|$p"$'\n' + done < <(_json_external_entries "$json") + fi + if [ -e "$legacy" ]; then + if [ -L "$legacy" ]; then + echo "WARNING: $legacy is a symlink; ignoring." >&2 + elif [ -f "$legacy" ]; then + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + case "$seen" in *"$t|$p"$'\n'*) continue ;; esac + printf '%s\t%s\t%s\n' "$t" "$p" "$k" + done < <(_parse_secrets_files_manifest "$legacy") + fi + fi +} + +# External tuples for PULL: the manifest wins entirely when present; +# legacy .secrets-files is only consulted in manifest-less projects. +_external_entries_for_pull() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + if [ -f "$json" ] && [ ! -L "$json" ]; then + if [ -f "$legacy" ] && [ ! -L "$legacy" ]; then + echo "WARNING: $legacy is superseded by $SECRETS_JSON_NAME and was ignored on pull. Run 'secrets push' to absorb it, then delete it." >&2 + fi + _json_external_entries "$json" + return 0 + fi + [ -e "$legacy" ] || return 0 + if [ -L "$legacy" ]; then + echo "WARNING: $legacy is a symlink; ignoring." >&2 + return 0 + fi + [ -f "$legacy" ] && _parse_secrets_files_manifest "$legacy" + return 0 +} + +# JSON array of legacy .secrets-files entries NOT yet in the manifest — +# what cmd_push absorbs. gradle-properties becomes 'properties' on the +# JSON side. Parser warnings suppressed (push_external_files re-parses +# and warns once). +_legacy_absorb_json() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + local out="[]" + if [ ! -f "$legacy" ] || [ -L "$legacy" ]; then + printf '%s' "$out" + return 0 + fi + local seen="" + if [ -f "$json" ] && [ ! -L "$json" ]; then + seen=$(jq -r '.external // [] | .[] | ((if .type == "properties" then "gradle-properties" else .type end) + "|" + .path)' "$json") + fi + local t p k s found jtype + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + found=0 + while IFS= read -r s; do [ "$s" = "$t|$p" ] && { found=1; break; }; done <<< "$seen" + [ "$found" -eq 1 ] && continue + jtype="$t"; [ "$t" = "gradle-properties" ] && jtype="properties" + out=$(printf '%s' "$out" | jq --arg type "$jtype" --arg path "$p" --arg keys "$k" \ + '. + [if $type == "file" then {type: $type, path: $path} + else {type: $type, path: $path, keys: ($keys | split(" ") | map(select(length > 0)))} end]') + done < <(_parse_secrets_files_manifest "$legacy" 2>/dev/null) + printf '%s' "$out" +} + # Quietly emit "ws-dir/basename" for every env file in a package.json # workspace under . Emits nothing (and never dies) when is # not a workspace monorepo or jq is unavailable — plain `push` calls this @@ -1178,20 +1309,39 @@ cmd_push() { fi # ── Manifest write AFTER successful encryption (bootstrap ordering) ── - if [ "$count" -gt 0 ] && [ -n "$to_add" ] && [ "$frozen" = false ] \ - && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then - local add_json - add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') + # Two independent reasons to write: dotenv auto-adds, and absorbing a + # legacy .secrets-files (gradle-properties → properties) so the two + # external sources converge on the manifest. + local absorbed_json="[]" n_absorbed=0 + if [ "$frozen" = false ]; then + absorbed_json=$(_legacy_absorb_json "$PWD") + n_absorbed=$(printf '%s' "$absorbed_json" | jq 'length') + fi + local write_adds=false + if [ -n "$to_add" ] && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then + write_adds=true + fi + if [ "$did" -eq 1 ] && [ "$frozen" = false ] \ + && { [ "$write_adds" = true ] || [ "$n_absorbed" -gt 0 ]; }; then + local add_json="[]" + [ "$write_adds" = true ] && add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') if [ "$have_manifest" = true ]; then - jq --argjson add "$add_json" '.dotenv = ((.dotenv // []) + $add)' "$manifest" \ + jq --argjson add "$add_json" --argjson ext "$absorbed_json" \ + '.dotenv = ((.dotenv // []) + $add) | .external = ((.external // []) + $ext)' "$manifest" \ | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" else - jq -n --argjson add "$add_json" '{version: '"$MANIFEST_VERSION"', dotenv: $add}' \ + jq -n --argjson add "$add_json" --argjson ext "$absorbed_json" \ + '{version: '"$MANIFEST_VERSION"', dotenv: $add} | if ($ext | length) > 0 then .external = $ext else . end' \ | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" fi - while IFS= read -r e; do - [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" - done <<< "$to_add" + if [ "$write_adds" = true ]; then + while IFS= read -r e; do + [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" + done <<< "$to_add" + fi + if [ "$n_absorbed" -gt 0 ]; then + info "Absorbed $n_absorbed entr(y/ies) from $SECRETS_FILES_NAME into $SECRETS_JSON_NAME (gradle-properties → properties). $SECRETS_FILES_NAME can be deleted." + fi info "Commit the manifest so other machines pick it up. To undo an entry: edit $SECRETS_JSON_NAME (or use 'secrets push --frozen' to skip auto-add)." fi diff --git a/test/manifest.bats b/test/manifest.bats index 9800c72..d9e97ae 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -244,3 +244,138 @@ load test_helper [ "$status" -eq 1 ] [[ "$output" == *"project-relative"* ]] || false } + +# ─── C: legacy absorb + external entries via .secrets.json ───────────── + +# Local fixtures (mirror secrets.bats EGB-531/652 helpers) +m_gradle_src() { mkdir -p "$HOME/.gradle"; printf '%s' "$1" > "$HOME/.gradle/gradle.properties"; } +m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > "$HOME/keystores/upload.keystore"; } + +@test "push absorbs .secrets-files into .secrets.json (properties + file)" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + m_file_src + local dir="$WORK_DIR/absorbproj"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\nfile ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"Absorbed"* ]] || false + run jq -r '.external | length' .secrets.json + [ "$output" = "2" ] + run jq -r '.external[] | select(.path == "~/.gradle/gradle.properties") | .type' .secrets.json + [ "$output" = "properties" ] + run jq -r '.external[] | select(.type == "file") | .path' .secrets.json + [ "$output" = "~/keystores/upload.keystore" ] + # stage 1: blob naming stays legacy-compatible + run bash -c "ls $SECRETS_DIR/absorbproj/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "absorb is idempotent — second push adds no duplicate externals" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/absorb2"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + run jq -r '.external | length' .secrets.json + [ "$output" = "1" ] +} + +@test "external properties entry in .secrets.json drives push without .secrets-files" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/jsonextproj"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push jsonextproj + [ "$status" -eq 0 ] + [[ "$output" == *"Extracted 1 key"* ]] || false + run bash -c "ls $SECRETS_DIR/jsonextproj/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "pull merges properties keys sourced from .secrets.json" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\nunrelated=keep\n' + local dir="$WORK_DIR/jsonpull"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + "$SECRETS_BIN" push jsonpull >/dev/null 2>&1 + m_gradle_src $'beaconClerkPkTest=STALE\nunrelated=keep\n' + run "$SECRETS_BIN" pull jsonpull + [ "$status" -eq 0 ] + run grep -c 'beaconClerkPkTest=pk_test_abc' "$HOME/.gradle/gradle.properties" + [ "$output" = "1" ] + run grep -c 'unrelated=keep' "$HOME/.gradle/gradle.properties" + [ "$output" = "1" ] +} + +@test "properties rail generalized: any *.properties basename is accepted" { + init_with_remote + mkdir -p "$HOME/.config" + printf 'apiKey=abc123\n' > "$HOME/.config/app.properties" + local dir="$WORK_DIR/genprops"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.config/app.properties","keys":["apiKey"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push genprops + [ "$status" -eq 0 ] + [[ "$output" == *"Extracted 1 key"* ]] || false +} + +@test "properties rail still blocks a non-.properties target" { + init_with_remote + printf 'PATH=/evil\n' > "$HOME/.bashrc" + local dir="$WORK_DIR/evilprops"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.bashrc","keys":["PATH"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push evilprops + [ "$status" -eq 1 ] + [[ "$output" == *".properties"* ]] || false +} + +@test "file entry via .secrets.json round-trips binary with mode 600" { + init_with_remote + m_file_src + local dir="$WORK_DIR/jsonfile"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"file","path":"~/keystores/upload.keystore"}]}\n' > "$dir/.secrets.json" + cd "$dir" + "$SECRETS_BIN" push jsonfile >/dev/null 2>&1 + cp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.keystore" + rm "$HOME/keystores/upload.keystore" + run "$SECRETS_BIN" pull jsonfile + [ "$status" -eq 0 ] + cmp -s "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.keystore" + local mode + mode=$(stat -f '%Lp' "$HOME/keystores/upload.keystore" 2>/dev/null || stat -c '%a' "$HOME/keystores/upload.keystore") + [ "$mode" = "600" ] +} + +@test "json file entry with keys is rejected with a warning" { + init_with_remote + m_file_src + local dir="$WORK_DIR/badfile"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"file","path":"~/keystores/upload.keystore","keys":["nope"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push badfile + [ "$status" -eq 0 ] + [[ "$output" == *"no keys"* ]] || false + run bash -c "ls $SECRETS_DIR/badfile/external/*.file.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "pull warns that .secrets-files is superseded when .secrets.json exists" { + init_with_remote + create_project_dir superproj + "$SECRETS_BIN" push superproj >/dev/null 2>&1 + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + run "$SECRETS_BIN" pull superproj + [ "$status" -eq 0 ] + [[ "$output" == *"superseded"* ]] || false +} diff --git a/test/secrets.bats b/test/secrets.bats index 56f051e..5537a03 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1217,15 +1217,17 @@ gradle_project() { [ "$mode" = "600" ] } -@test "EGB-531: target with wrong basename is refused" { +@test "EGB-531: target with non-.properties basename is refused" { + # EGB-677 generalized the rail from exact 'gradle.properties' to any + # '*.properties' basename — shell rc files and gitconfig stay blocked. init_with_remote mkdir -p "$HOME/.gradle" - printf 'beaconClerkPkTest=x\n' > "$HOME/.gradle/custom.properties" + printf 'beaconClerkPkTest=x\n' > "$HOME/.gradle/evil.sh" mkdir -p "$WORK_DIR/gproj"; cd "$WORK_DIR/gproj" - printf 'gradle-properties ~/.gradle/custom.properties beaconClerkPkTest\n' > .secrets-files + printf 'gradle-properties ~/.gradle/evil.sh beaconClerkPkTest\n' > .secrets-files run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"gradle.properties"* ]] || false + [[ "$output" == *".properties"* ]] || false } @test "EGB-531: target outside HOME is refused" { @@ -1404,6 +1406,9 @@ gradle_project() { init_with_remote create_project_dir gproj "$SECRETS_BIN" push gproj >/dev/null 2>&1 + # EGB-677: drop the bootstrap .secrets.json so the legacy manifest path + # is exercised (with a manifest present, .secrets-files is superseded). + rm -f "$WORK_DIR/gproj/.secrets.json" printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$WORK_DIR/gproj/.secrets-files" cd "$WORK_DIR/gproj" run "$SECRETS_BIN" pull gproj From 0049584d9b90ae70d3ace7d96ebfe6c4367c002f Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:49:02 -0700 Subject: [PATCH 05/70] =?UTF-8?q?feat:=20manifest-driven=20pull=20?= =?UTF-8?q?=E2=80=94=20nested=20restore,=20restore-time=20rail,=20empty=20?= =?UTF-8?q?no-op=20(EGB-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pull with .secrets.json restores exactly the declared entries (nested paths get mkdir -p); stray store blobs are not restored - dotenv rail re-runs at restore time: unsafe entries warn+skip (pull never dies on one bad entry), missing blobs warn with a directed hint - empty manifest = warn no-op instead of a confusing 'not found' death - manifest-less projects keep the legacy glob pull verbatim --- secrets | 48 +++++++++++++++++++++++++++ test/manifest.bats | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/secrets b/secrets index e02e636..1e75d98 100755 --- a/secrets +++ b/secrets @@ -1414,6 +1414,54 @@ cmd_pull() { git -C "$SECRETS_DIR" pull >/dev/null 2>&1 fi + # ── Manifest-driven pull (EGB-677 stage 1) ── + # With a .secrets.json present, the manifest decides what restores and + # where (nested entries get their directories created). The dotenv rail + # runs again at restore time — warn+skip on pull, never die, so one bad + # entry can't block the rest of the restore. + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + local declared n_external + declared=$(jq -r '.dotenv // [] | .[]' "$manifest") + n_external=$(jq -r '.external // [] | length' "$manifest") + + if [ -z "$declared" ] && [ "$n_external" -eq 0 ]; then + echo "WARNING: $SECRETS_JSON_NAME declares nothing to pull (empty manifest). Run 'secrets push' on a machine that has the files." >&2 + ensure_store_protections + return 0 + fi + if [ -n "$declared" ] && [ ! -d "$SECRETS_DIR/$project" ]; then + die "Project '$project' not found. Run: secrets list" + fi + + local count=0 rel + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then + echo "WARNING: skipping unsafe dotenv path from $SECRETS_JSON_NAME: $rel" >&2 + continue + fi + local blob="$SECRETS_DIR/$project/${rel}.age" + if [ ! -f "$blob" ]; then + echo "WARNING: '$rel' is declared in $SECRETS_JSON_NAME but has no encrypted data in the store yet. Run 'secrets push' on a machine that has it. Skipping." >&2 + continue + fi + case "$rel" in */*) mkdir -p "$target_dir/$(dirname "$rel")" ;; esac + age -d -i "$KEY_FILE" -o "$target_dir/$rel" "$blob" + if [ ! -s "$target_dir/$rel" ]; then + echo "WARNING: Decrypted file '$rel' is empty (possibly truncated .age blob)" + fi + count=$((count + 1)) + done <<< "$declared" + + info "Decrypted $count file(s) into $target_dir" + pull_external_files "$PWD" "$project" + ensure_store_protections + return 0 + fi + + # ── Legacy glob pull (manifest-less projects; unchanged) ── # Check project exists if [ ! -d "$SECRETS_DIR/$project" ]; then die "Project '$project' not found. Run: secrets list" diff --git a/test/manifest.bats b/test/manifest.bats index d9e97ae..f23a5aa 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -379,3 +379,86 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -eq 0 ] [[ "$output" == *"superseded"* ]] || false } + +# ─── D: pull from manifest — nested restore, restore-time rail ───────── + +@test "pull restores manifest-declared nested file (mkdir -p)" { + init_with_remote + create_project_dir nestpull + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + rm -rf packages + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [ -f packages/web/.env.development ] + [ "$(cat packages/web/.env.development)" = "K=v" ] +} + +@test "pull with manifest restores only declared entries" { + init_with_remote + create_project_dir onlydecl + "$SECRETS_BIN" push >/dev/null 2>&1 + # plant an undeclared stray blob in the store + local pubkey; pubkey=$(age-keygen -y "$SECRETS_DIR/key.txt") + echo "S=1" | age -r "$pubkey" -o "$SECRETS_DIR/onlydecl/.env.stray.age" + rm -f .env .env.staging + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [ -f .env ] + [ ! -f .env.stray ] +} + +@test "pull warns and skips an unsafe manifest entry, restores the rest" { + init_with_remote + create_project_dir unsafepull + "$SECRETS_BIN" add .env >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + printf '{"version":2,"dotenv":[".env","../escape/.env"]}\n' > .secrets.json + rm -f .env + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [[ "$output" == *"skipping unsafe"* ]] || false + [ -f .env ] + [ ! -f "$WORK_DIR/escape/.env" ] +} + +@test "pull on an empty manifest is a warn no-op" { + init_with_remote + mkdir -p "$WORK_DIR/emptypull" + cd "$WORK_DIR/emptypull" + printf '{"version":2,"dotenv":[]}\n' > .secrets.json + run "$SECRETS_BIN" pull emptypull + [ "$status" -eq 0 ] + [[ "$output" == *"declares nothing"* ]] || false +} + +@test "pull warns when a declared entry has no blob in the store" { + init_with_remote + create_project_dir nopullblob + "$SECRETS_BIN" push >/dev/null 2>&1 + jq '.dotenv += [".env.missing"]' .secrets.json > .secrets.json.tmp && mv .secrets.json.tmp .secrets.json + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [[ "$output" == *".env.missing"* ]] || false + [[ "$output" == *"no encrypted data"* ]] || false +} + +@test "machine-2 flow: committed manifest + pull restores everything" { + init_with_remote + create_project_dir machine1 + mkdir -p packages/api + echo "API=1" > packages/api/.dev.vars + "$SECRETS_BIN" add packages/api/.dev.vars >/dev/null + "$SECRETS_BIN" push m2proj >/dev/null 2>&1 + # simulate machine 2: fresh dir, only the committed manifest present + mkdir -p "$WORK_DIR/machine2" + cp .secrets.json "$WORK_DIR/machine2/" + cd "$WORK_DIR/machine2" + run "$SECRETS_BIN" pull m2proj + [ "$status" -eq 0 ] + [ -f .env ] + [ -f packages/api/.dev.vars ] + [ "$(cat packages/api/.dev.vars)" = "API=1" ] +} From 89e851278b2db645591975827d46c9db7c538408 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:58:16 -0700 Subject: [PATCH 06/70] feat: jq gating, platform-aware install hints, stage-1 docs (EGB-677 stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jq required only when .secrets.json exists or is being written; manifest-less projects run jq-free (features skipped with a notice) — clone-and-run for v1 users survives (no-jq fixture excludes /usr/bin, macOS ships jq there now) - check_cmd: platform-aware hints (brew/apt-get/dnf/generic) instead of hardcoded brew — correct guidance on Linux/CI - cmd_help: add command, push flags, manifest section with example - README: manifest section, external files rewritten around .secrets.json (legacy .secrets-files documented as absorbed), troubleshooting entries, command table, test instructions - CLAUDE.md: manifest architecture notes, bash-3.2 '[[ ]] || false' testing convention, project structure refresh --- CLAUDE.md | 13 ++++++-- README.md | 77 +++++++++++++++++++++++++++++++++++++--------- secrets | 55 ++++++++++++++++++++++++++++++--- test/manifest.bats | 45 +++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0da5504..ede0401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,9 +16,14 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core -bats test/secrets.bats +bats test/ # runs secrets.bats + manifest.bats ``` +**bash 3.2 assertion gotcha:** bats runs under system bash 3.2, where a +failing `[[ ]]` mid-test does NOT fail the test (the ERR trap skips `[[` +compound commands). Every standalone `[[ ... ]]` assertion MUST end with +`|| false`. Single-bracket `[ ]` assertions are unaffected. + ## Architecture Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. @@ -26,6 +31,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) +- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` - Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`) @@ -34,10 +40,11 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek ## Project Structure ``` -secrets # CLI script (~600 lines bash) +secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ - secrets.bats # bats-core test suite (126 tests) + secrets.bats # bats-core test suite (133 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (41 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index d68e566..f7e754d 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,10 @@ secrets clear |---------|-------------| | `secrets init` | Create the `~/.secrets/` repo and generate an encryption key | | `secrets push` | Encrypt secret files in the current directory and upload them | +| `secrets push --frozen` | Sync only what `.secrets.json` declares (skip auto-add) | +| `secrets push --dry-run` | Show what would be added/synced without changing anything | | `secrets pull` | Download and decrypt secret files into the current directory | +| `secrets add ` | Declare a project-relative file in `.secrets.json` | | `secrets clear` | Delete plaintext secret files from the current directory | | `secrets run ` | Pull secrets, run a command, then clear secrets when it exits | | `secrets list` | Show all projects that have stored secrets | @@ -176,6 +179,32 @@ When you run `secrets push` or `secrets pull` without specifying a project name, You can also specify a name explicitly: `secrets push myapp`. +### The manifest (`.secrets.json`) + +Every project gets a committed `.secrets.json` at its root declaring exactly what syncs — the single source of truth `push` and `pull` operate from (requires `jq`): + +```json +{ + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", ".env.staging", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] +} +``` + +You rarely write it by hand: + +- **Auto-add (default):** `secrets push` discovers conventional files (`.env`, `.env.*`, `.dev.vars` — plus `package.json` workspace dirs once a manifest exists) and adds them to the manifest with an `==>` notice. Commit the manifest so other machines pick it up. +- **Explicit mode:** set `"options": {"autoAdd": false}` (a committed, team-shared setting) and `push` only syncs declared entries, warning about undeclared files. `secrets add ` is then the only manifest writer. Per-invocation: `push --frozen` (declared-only once) and `push --dry-run` (preview). +- `dotenv` paths are project-relative — nested monorepo paths like `packages/@acme/web/.env` are welcome; `..`, absolute paths, and symlinked manifests are refused. +- On the other machine, `secrets pull` restores exactly what the committed manifest declares, creating nested directories as needed. + +Projects without a manifest keep working exactly as before (and work without `jq`); the first `push` bootstraps one for you. + ### secrets run `secrets run` is a **pull → run → clear** pipeline: it runs `secrets pull` to decrypt the latest files into your project, executes your command, then runs `secrets clear` when that command finishes. Plaintext `.env` / `.dev.vars` files exist only while your command is running. @@ -352,25 +381,40 @@ Requires `jq` (`brew install jq`). Some credentials don't live in your project at all. Android builds, for example, read keys from `~/.gradle/gradle.properties` — a global file, outside any project, shared by every Gradle project on your machine (the project's own `gradle.properties` is git-tracked, so it's the wrong home for secrets). `secrets` can sync specific keys from such a file without touching the unrelated keys around them. -You declare what to sync in a committed `.secrets-files` manifest at your project root, one entry per line: +You declare what to sync in the `external` array of your committed `.secrets.json`: -``` -# -gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive +```json +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} ``` -- **type** — `gradle-properties` (sync named keys) or `file` (sync the whole file — see below). -- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `gradle-properties` the basename must be `gradle.properties`. -- **keys** — the property names to sync (`gradle-properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. +- **type** — `properties` (sync named keys from a Java-properties-style file) or `file` (sync the whole file — see below). +- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `properties` the basename must end in `.properties`. +- **keys** — the property names to sync (`properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. + +> **Legacy `.secrets-files`:** older projects declared these entries in a line-based `.secrets-files`. It still parses, and the next `secrets push` absorbs its entries into `.secrets.json` (type `gradle-properties` becomes `properties`) — after that the legacy file is superseded and can be deleted. #### Syncing to a second machine -On the machine that already has the keys set: +On the machine that already has the keys set, add the entry to `.secrets.json` (create the file if the project doesn't have one yet): ```bash cd ~/myapp -echo "gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive" > .secrets-files -git add .secrets-files && git commit -m "sync gradle Clerk keys" +cat > .secrets.json <<'EOF' +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} +EOF +git add .secrets.json && git commit -m "sync gradle Clerk keys" secrets push # ==> Extracted 2 key(s) from ~/.gradle/gradle.properties ``` @@ -393,9 +437,8 @@ secrets pull Some external secrets are whole binary files — an Android upload keystore, a certificate. The `file` type syncs the file verbatim (binary-safe, encrypted with age like everything else): -``` -# -file ~/keystores/beacon-upload.keystore +```json +{ "type": "file", "path": "~/keystores/beacon-upload.keystore" } ``` On `secrets push` the file is encrypted into `/external/`. On `secrets pull` it is restored to the same path with mode `600`; if a different version already exists there, it is backed up to `.secrets-bak` first. The same path rules apply (inside `$HOME`, no `..`, no symlinks). Like merged Gradle keys, restored files are permanent plaintext on disk — `secrets clear` does not remove them. @@ -444,10 +487,14 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` **"Fast-forward pull failed"** — Someone else pushed secrets while you had local changes. Run `secrets pull` first, then retry your push. +**".secrets.json: invalid JSON" / "manifest version N is not supported"** — The committed manifest is malformed or written by a newer `secrets`. The error names the file; fix the syntax, or update the tool (`git pull` in the tool's clone). + +**"'jq' is not installed"** — Manifest features need `jq`. The error prints the install command for your platform. Manifest-less projects work without it. + ## Development ```bash -# Run the test suite (126 tests) +# Run the test suite (174 tests across both files) brew install bats-core -bats test/secrets.bats +bats test/ ``` diff --git a/secrets b/secrets index 1e75d98..7e01e91 100755 --- a/secrets +++ b/secrets @@ -37,7 +37,18 @@ die() { echo "ERROR: $*" >&2; exit 1; } info() { echo "==> $*"; } check_cmd() { - command -v "$1" >/dev/null 2>&1 || die "'$1' is not installed. Run: brew install $1" + command -v "$1" >/dev/null 2>&1 && return + # Platform-aware install hint — hardcoding brew is wrong guidance on a + # Linux box or CI runner (EGB-677 DX review). + local hint="install '$1' with your package manager" + if command -v brew >/dev/null 2>&1; then + hint="brew install $1" + elif command -v apt-get >/dev/null 2>&1; then + hint="sudo apt-get install $1" + elif command -v dnf >/dev/null 2>&1; then + hint="sudo dnf install $1" + fi + die "'$1' is not installed. Run: $hint" } check_initialized() { @@ -1216,6 +1227,12 @@ cmd_push() { pubkey=$(get_pubkey) # ── Manifest read (validated; absence = bootstrap) ── + # jq is required only when a manifest exists (authoritative, can't be + # ignored) or is being written. Without jq on a manifest-less project, + # manifest features are skipped with a notice — clone-and-run for v1 + # users survives. + local have_jq=true + command -v jq >/dev/null 2>&1 || have_jq=false local manifest="$PWD/$SECRETS_JSON_NAME" local have_manifest=false auto_add=true declared="" if [ -e "$manifest" ]; then @@ -1312,8 +1329,11 @@ cmd_push() { # Two independent reasons to write: dotenv auto-adds, and absorbing a # legacy .secrets-files (gradle-properties → properties) so the two # external sources converge on the manifest. + if [ "$have_jq" = false ]; then + echo "NOTE: jq not found — skipping $SECRETS_JSON_NAME manifest features (auto-add, absorb). Install jq to enable them." >&2 + fi local absorbed_json="[]" n_absorbed=0 - if [ "$frozen" = false ]; then + if [ "$frozen" = false ] && [ "$have_jq" = true ]; then absorbed_json=$(_legacy_absorb_json "$PWD") n_absorbed=$(printf '%s' "$absorbed_json" | jq 'length') fi @@ -1321,7 +1341,7 @@ cmd_push() { if [ -n "$to_add" ] && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then write_adds=true fi - if [ "$did" -eq 1 ] && [ "$frozen" = false ] \ + if [ "$did" -eq 1 ] && [ "$frozen" = false ] && [ "$have_jq" = true ] \ && { [ "$write_adds" = true ] || [ "$n_absorbed" -gt 0 ]; }; then local add_json="[]" [ "$write_adds" = true ] && add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') @@ -1905,21 +1925,48 @@ secrets — encrypted secret file sync between machines Usage: secrets init Initialize the secrets repo and generate an age key secrets push [project] Encrypt secret files and push to the secrets repo + secrets push --frozen Sync only manifest-declared files (skip auto-add) + secrets push --dry-run Show what would be added/synced; change nothing secrets push -w|--workspaces Push secrets from all workspaces in package.json secrets pull [project] Pull and decrypt secret files into current directory secrets pull -w|--workspaces Pull secrets into all workspaces from package.json + secrets add Declare a project-relative file in .secrets.json secrets clear Remove plaintext secret files from current directory secrets clear -w|--workspaces Clear secrets from all workspaces in package.json secrets run [-w] Pull secrets, run command, clear secrets on exit secrets list List all projects and their secret files secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key - secrets which Show the active store path and which rule chose it + secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` Tracked files: .env, .env.*, .dev.vars +Manifest (.secrets.json): + A committed project-root manifest declaring everything the project + syncs (requires jq). `secrets push` discovers conventional files and + auto-adds them with a notice; set {"options":{"autoAdd":false}} to + require explicit `secrets add` instead. Dotenv paths are project- + relative (nested workspace paths welcome); external entries use + {"type":"properties"|"file","path":...,"keys":[...]}: + + { + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] + } + + A legacy .secrets-files is absorbed into .secrets.json on the next + push (gradle-properties entries become type "properties") and can be + deleted afterwards. Without jq, manifest-less projects keep working; + manifest features are skipped with a notice. + If [project] is omitted, it is derived from the current directory's git remote (if available) or the directory name. diff --git a/test/manifest.bats b/test/manifest.bats index f23a5aa..822c1ca 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -462,3 +462,48 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ -f packages/api/.dev.vars ] [ "$(cat packages/api/.dev.vars)" = "API=1" ] } + +# ─── E: jq gating + install hints + help ─────────────────────────────── + +# Helper: PATH with age but without jq. macOS ships /usr/bin/jq, so +# /usr/bin must be excluded too — needed tools are symlinked explicitly. +m_nojq_path() { + local fake="$TEST_TMPDIR/nojq-bin" + mkdir -p "$fake" + local t + for t in age age-keygen git basename dirname mktemp grep sed tr cut cksum stat head tail sort uniq wc env touch find diff cmp; do + command -v "$t" >/dev/null 2>&1 && ln -sf "$(command -v "$t")" "$fake/$t" + done + rm -f "$fake/jq" + echo "$fake:/bin" +} + +@test "manifest-less push works without jq (manifest features skipped)" { + init_with_remote + create_project_dir nojqproj + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/nojqproj/.env.age" ] + [ ! -f ".secrets.json" ] + [[ "$output" == *"jq"* ]] || false +} + +@test "push dies with an install hint when a manifest exists but jq is missing" { + init_with_remote + create_project_dir needjq + printf '{"version":2,"dotenv":[".env"]}\n' > .secrets.json + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [[ "$output" == *"'jq' is not installed"* ]] || false +} + +@test "help documents add, --frozen, --dry-run and the manifest" { + run "$SECRETS_BIN" help + [ "$status" -eq 0 ] + [[ "$output" == *"secrets add"* ]] || false + [[ "$output" == *"--frozen"* ]] || false + [[ "$output" == *"--dry-run"* ]] || false + [[ "$output" == *".secrets.json"* ]] || false +} From ecc4f2349484e719a6ae8b18d1ef7f7bc9bffc3b Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 09:40:43 -0700 Subject: [PATCH 07/70] test: coverage for stage-1 gap paths (json rail warns, symlink legacy, directed errors, dry-run/which branches) --- test/manifest.bats | 115 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/test/manifest.bats b/test/manifest.bats index 822c1ca..c4d7314 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -507,3 +507,118 @@ m_nojq_path() { [[ "$output" == *"--dry-run"* ]] || false [[ "$output" == *".secrets.json"* ]] || false } + +# ─── Coverage backfill (ship Step 7 gap paths) ───────────────────────── + +@test "json external entry with unknown type warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unktype"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"wat","path":"~/x.properties","keys":["k"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unktype + [ "$status" -eq 0 ] + [[ "$output" == *"unknown external type"* ]] || false +} + +@test "json external entry with unsafe path warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unsafext"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/../etc/x.properties","keys":["k"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unsafext + [ "$status" -eq 0 ] + [[ "$output" == *"unsafe characters in external path"* ]] || false +} + +@test "json external entry with unsafe keys warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unsafekeys"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["k;rm"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unsafekeys + [ "$status" -eq 0 ] + [[ "$output" == *"unsafe characters in key list"* ]] || false +} + +@test "json properties entry without keys warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/nokeys"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties"}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push nokeys + [ "$status" -eq 0 ] + [[ "$output" == *"has no keys"* ]] || false +} + +@test "symlinked .secrets-files is ignored with a warning on push" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=x\n' + local dir="$WORK_DIR/symlegacy"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$TEST_TMPDIR/real-sf" + ln -s "$TEST_TMPDIR/real-sf" "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push symlegacy + [ "$status" -eq 0 ] + [[ "$output" == *"symlink"* ]] || false + run bash -c "ls $SECRETS_DIR/symlegacy/external/*.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "pull with declared entries dies with directed error when project absent from store" { + init_with_remote + mkdir -p "$WORK_DIR/ghostproj" + cd "$WORK_DIR/ghostproj" + printf '{"version":2,"dotenv":[".env"]}\n' > .secrets.json + run "$SECRETS_BIN" pull ghostproj + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] || false + [[ "$output" == *"secrets list"* ]] || false +} + +@test "add without an argument dies with usage" { + create_project_dir noargadd + run "$SECRETS_BIN" add + [ "$status" -eq 1 ] + [[ "$output" == *"Usage: secrets add"* ]] || false +} + +@test "push rejects an unknown flag with usage" { + init_with_remote + create_project_dir badflag + run "$SECRETS_BIN" push --nope + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown push flag"* ]] || false +} + +@test "add normalizes a leading ./ prefix" { + create_project_dir dotslash + run "$SECRETS_BIN" add ./.env + [ "$status" -eq 0 ] + run jq -r '.dotenv[0]' .secrets.json + [ "$output" = ".env" ] +} + +@test "push --dry-run reports nothing-new when manifest covers all discovered files" { + init_with_remote + create_project_dir alldecl + "$SECRETS_BIN" add .env >/dev/null + "$SECRETS_BIN" add .env.staging >/dev/null + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Nothing new to add"* ]] || false +} + +@test "which displays external entries from the manifest" { + create_project_dir whichext + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["k1"]}]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"properties"* ]] || false + [[ "$output" == *"gradle.properties"* ]] || false + [[ "$output" == *"k1"* ]] || false +} From 5489f894460fb1329d54073cbd134b71e7332e5d Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 10:17:07 -0700 Subject: [PATCH 08/70] =?UTF-8?q?test:=20coverage=20backfill=20=E2=80=94?= =?UTF-8?q?=20which=20UNSAFE=20marker,=20malformed-manifest=20add,=20dry-r?= =?UTF-8?q?un=20declared=20list,=20frozen=20absorb=20suppression,=20file-t?= =?UTF-8?q?ype=20absorb=20round-trip,=20external=20dedup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/manifest.bats | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/test/manifest.bats b/test/manifest.bats index c4d7314..e273479 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -622,3 +622,79 @@ m_nojq_path() { [[ "$output" == *"gradle.properties"* ]] || false [[ "$output" == *"k1"* ]] || false } + +# ─── F: ship Step 7 coverage backfill (audit gaps) ───────────────────── + +@test "which flags an unsafe dotenv entry with the UNSAFE marker" { + create_project_dir whichunsafe + printf '{"version":2,"dotenv":[".env","../escape/.env"]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"UNSAFE"* ]] || false + [[ "$output" == *"will be refused"* ]] || false +} + +@test "add to a malformed existing manifest dies with a directed error" { + create_project_dir addmalformed + echo '{ not json' > .secrets.json + run "$SECRETS_BIN" add .env + [ "$status" -eq 1 ] + [[ "$output" == *"invalid"* ]] || false +} + +@test "push --dry-run lists declared entries under 'Would sync'" { + init_with_remote + create_project_dir drysync + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Would sync (declared)"* ]] || false + [[ "$output" == *".env"* ]] || false +} + +@test "push --frozen does not absorb a legacy .secrets-files" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/frozenabsorb"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"dotenv":[".env"]}\n' > "$dir/.secrets.json" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push --frozen frozenabsorb + [ "$status" -eq 0 ] + [[ "$output" != *"Absorbed"* ]] || false + run jq -r '.external // [] | length' .secrets.json + [ "$output" = "0" ] +} + +@test "file entry absorbed from legacy round-trips on pull" { + init_with_remote + m_file_src + local dir="$WORK_DIR/fileabsorb"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'file ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + "$SECRETS_BIN" push fileabsorb >/dev/null 2>&1 + run jq -r '.external[] | select(.type=="file") | .path' .secrets.json + [ "$output" = "~/keystores/upload.keystore" ] + cp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.ks" + rm "$HOME/keystores/upload.keystore" + rm -f .secrets-files + run "$SECRETS_BIN" pull fileabsorb + [ "$status" -eq 0 ] + cmp -s "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.ks" +} + +@test "push dedups a legacy entry already present in the manifest external[]" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/dedupext"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push dedupext + [ "$status" -eq 0 ] + run jq -r '.external | length' .secrets.json + [ "$output" = "1" ] +} From c6ea724ddbd36b002b75a15bc1a1497c04fcd9f6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:12:27 -0700 Subject: [PATCH 09/70] fix: rekey and list recurse into nested manifest blobs (EGB-677 stage 1) Pre-landing review (testing + checklist specialists, reproduced) caught a data-loss bug: cmd_rekey's decrypt/re-encrypt globs were non-recursive and only special-cased external/. Nested manifest dotenv blobs (/.age, new this branch) were never visited, so after a key rotation they stayed encrypted under the discarded old key = permanently undecryptable. cmd_list had the same blind spot (cosmetic: nested entries invisible in listings). Both now walk the entire project tree with `find -type f` (bash 3.2 safe, includes dotfiles natively), unifying top-level / nested / external blobs into one recursive pass and dropping the now-redundant external/ special-casing. Regression tests: nested-blob rekey round-trip (survives rotation) + list shows nested entry. Full suite 193/193. --- secrets | 85 ++++++++++++++++++++-------------------------- test/manifest.bats | 33 ++++++++++++++++++ 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/secrets b/secrets index 7e01e91..ba2adce 100755 --- a/secrets +++ b/secrets @@ -1607,19 +1607,20 @@ cmd_list() { # Skip hidden dirs [[ "$project" == .* ]] && continue echo "$project:" - for f in "$dir"*.age "$dir".*.age; do + # Recurse the whole project tree so nested manifest blobs + # (/.age) are visible, not just top-level entries. + # External blobs (external/.age) are labelled distinctly. + while IFS= read -r f; do [ -f "$f" ] || continue - echo " $(basename "$f" .age)" + local rel + rel=${f#"$dir"} + rel=${rel%.age} + case "$rel" in + external/*) echo " [external] ${rel#external/}" ;; + *) echo " $rel" ;; + esac found=1 - done - # External files live in a subdir, invisible to the globs above. - if [ -d "${dir}external" ]; then - for f in "${dir}external"/*.age; do - [ -f "$f" ] || continue - echo " [external] $(basename "$f" .age)" - found=1 - done - fi + done < <(find "$dir" -type f -name '*.age' | sort) done if [ "$found" -eq 0 ]; then @@ -1681,29 +1682,23 @@ cmd_rekey() { project=$(basename "$dir") [[ "$project" == .* ]] && continue mkdir -p "$tmpdir/$project" - for f in "$dir"*.age "$dir".*.age; do + # Walk the WHOLE project tree, not just its top level. Manifest dotenv + # entries can nest (/.age) and external blobs live in + # /external/.age. A non-recursive glob would skip both, + # leaving them encrypted under the old key = permanently undecryptable + # after rotation (silent data loss). `find` is bash-3.2 safe and recurses. + while IFS= read -r f; do [ -f "$f" ] || continue - local name - name=$(basename "$f" .age) - if ! age -d -i "$KEY_FILE" -o "$tmpdir/$project/$name" "$f"; then - die "Decryption failed for $project/$name. Rekey aborted. Old key preserved." + local rel dest + rel=${f#"$dir"} # path relative to the project dir (keeps .age) + rel=${rel%.age} # strip the .age suffix → original relpath + dest="$tmpdir/$project/$rel" + mkdir -p "$(dirname "$dest")" + if ! age -d -i "$KEY_FILE" -o "$dest" "$f"; then + die "Decryption failed for $project/$rel. Rekey aborted. Old key preserved." fi file_count=$((file_count + 1)) - done - # External files live in a subdir; rekey them too or they become - # undecryptable after rotation. - if [ -d "${dir}external" ]; then - mkdir -p "$tmpdir/$project/external" - for f in "${dir}external"/*.age; do - [ -f "$f" ] || continue - local ename - ename=$(basename "$f" .age) - if ! age -d -i "$KEY_FILE" -o "$tmpdir/$project/external/$ename" "$f"; then - die "Decryption failed for $project/external/$ename. Rekey aborted. Old key preserved." - fi - file_count=$((file_count + 1)) - done - fi + done < <(find "$dir" -type f -name '*.age') done if [ "$file_count" -eq 0 ]; then @@ -1723,29 +1718,23 @@ cmd_rekey() { info "Re-encrypting all files with new key..." - # Re-encrypt all files. The ".*" glob is required: dotenv files decrypt - # to dotfiles ("$tmpdir/p/.env") that a bare "*" would silently skip, - # leaving their blobs on the old key (undecryptable after rotation). + # Re-encrypt all files. `find -type f` recurses into nested dotenv dirs and + # external/ and natively includes dotfiles (decrypted dotenv files like + # "$tmpdir/p/.env"), which a bare "*" glob would silently skip — leaving + # their blobs on the old key (undecryptable after rotation). The walk mirrors + # the recursive decrypt above so every blob round-trips back to its relpath. for dir in "$tmpdir"/*/; do [ -d "$dir" ] || continue local project project=$(basename "$dir") mkdir -p "$SECRETS_DIR/$project" - for f in "$dir"* "$dir".*; do + while IFS= read -r f; do [ -f "$f" ] || continue - local name - name=$(basename "$f") - age -r "$pubkey" -o "$SECRETS_DIR/$project/${name}.age" "$f" - done - if [ -d "${dir}external" ]; then - mkdir -p "$SECRETS_DIR/$project/external" - for f in "${dir}external"/*; do - [ -f "$f" ] || continue - local ename - ename=$(basename "$f") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/${ename}.age" "$f" - done - fi + local rel + rel=${f#"$dir"} # path relative to the project temp dir + mkdir -p "$(dirname "$SECRETS_DIR/$project/$rel")" + age -r "$pubkey" -o "$SECRETS_DIR/$project/${rel}.age" "$f" + done < <(find "$dir" -type f) done # Commit and push (heal .gitignore first so add -A can't stage key.txt) diff --git a/test/manifest.bats b/test/manifest.bats index e273479..4459a6c 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -698,3 +698,36 @@ m_nojq_path() { run jq -r '.external | length' .secrets.json [ "$output" = "1" ] } + +# ─── J: rekey + list recurse into nested manifest blobs (data-loss guard) ────── + +@test "rekey re-encrypts a nested manifest dotenv blob (survives rotation)" { + # Regression: cmd_rekey's non-recursive glob skipped /.age + # blobs, leaving them on the old key = permanently undecryptable after rotation. + init_with_remote + create_project_dir nestrekey + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + [ -f "$SECRETS_DIR/nestrekey/packages/web/.env.development.age" ] + run "$SECRETS_BIN" rekey + [ "$status" -eq 0 ] + rm -rf packages + run "$SECRETS_BIN" pull nestrekey + [ "$status" -eq 0 ] + [ -f packages/web/.env.development ] + [ "$(cat packages/web/.env.development)" = "N=nested" ] +} + +@test "list shows a nested manifest blob" { + init_with_remote + create_project_dir nestlist + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" == *"packages/web/.env.development"* ]] || false +} From 588f290dcc131c91b2ce1d6fc02463e43c36634c Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:12:39 -0700 Subject: [PATCH 10/70] chore: security review policy + operator runner, doc updates (EGB-677 stage 1) Add .ship-policy.json (opts out AI adversarial/red-team/security-specialist review; requires local operator sign-off) and test/run-security.sh (the operator-local security regression subset). Document the policy in CLAUDE.md and README, fix stale test counts (manifest.bats 41->58, total 174->191), and update the storage-recursion note to reflect rekey/list now walking the full project tree. --- .ship-policy.json | 15 ++++++++++ CLAUDE.md | 36 ++++++++++++++++++++++-- README.md | 9 +++++- test/run-security.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 .ship-policy.json create mode 100755 test/run-security.sh diff --git a/.ship-policy.json b/.ship-policy.json new file mode 100644 index 0000000..034532e --- /dev/null +++ b/.ship-policy.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "ai_review": { + "adversarial": false, + "red_team": false, + "security_specialist": false, + "reason": "This repo ships defensive security regression tests with attack-payload fixtures (shell injection, path traversal, symlink rails). AI-driven adversarial, red-team, and security-specialist review triggers Anthropic Usage Policy blocks (real-time cyber safeguards on hosted models). Do not dispatch those subagents or prompts for this repository." + }, + "manual_security_signoff": { + "script": "./test/run-security.sh", + "signoff_file": ".gstack/security-signoff.json", + "required_before_ship": true, + "instructions": "A human operator runs the script locally, reviews output, and types SIGNOFF. Agents must not run the security suite or substitute AI review for sign-off." + } +} diff --git a/CLAUDE.md b/CLAUDE.md index ede0401..174bf07 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,6 +17,7 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core bats test/ # runs secrets.bats + manifest.bats +./test/run-security.sh # security regression subset + operator sign-off (see below) ``` **bash 3.2 assertion gotcha:** bats runs under system bash 3.2, where a @@ -24,6 +25,35 @@ failing `[[ ]]` mid-test does NOT fail the test (the ERR trap skips `[[` compound commands). Every standalone `[[ ... ]]` assertion MUST end with `|| false`. Single-bracket `[ ]` assertions are unaffected. +## Security review policy (mandatory for agents) + +This repo contains **defensive** security regression tests with deliberate +attack-payload fixtures (shell injection strings, symlink escape cases, path +rails). Hosted models (including Opus) block AI-driven "think like an attacker" +/ red-team / security-specialist review of that material under Anthropic Usage +Policy (real-time cyber safeguards). + +**Read `.ship-policy.json` before any `/ship`, `/review`, or `/autoplan` work.** + +When that file is present (it always is in this repo): + +1. **Never** dispatch red-team subagents, security specialist subagents, or + adversarial review prompts ("think like an attacker", "find security holes", + "chaos engineer", exploit hunting) against this codebase. +2. **Never** run `./test/run-security.sh` or filter bats to SECURITY/adversarial + tests on behalf of the user — that is operator-local work only. +3. **Before ship/PR:** ask the human operator to run `./test/run-security.sh` + locally and complete the SIGNOFF prompt. Record lives at + `.gstack/security-signoff.json` (gitignored). Proceed with ship only after + they confirm sign-off for the current `HEAD` (or note in the PR that sign-off + is pending). +4. Normal `./ship` steps may continue (tests via `bats test/`, eng review, + changelog, PR) — only the AI security/red-team/adversarial passes are opted out. + +Gstack `/ship` reads the same policy via `gstack-ship-policy` (sources +`.ship-policy.json` at repo root): Step 9 blocks when manual sign-off is missing, +skips security specialist + red team, and Step 11 skips adversarial review. + ## Architecture Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. @@ -44,7 +74,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (41 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (58 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file @@ -74,7 +104,7 @@ The active store directory is picked by `resolve_store()` using these rules, hig Key design decisions (all driven by /autoplan review): - **Wire-in is at command scope** (`cmd_push`/`cmd_pull`), via `push_external_files` / `pull_external_files`, **not** inside `push_dir_to_project` / `pull_project_to_dir` (those loop per-workspace and `pull_project_to_dir` uses stdout as a data channel). -- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the existing non-recursive `*.age` / `.*.age` globs (pull, list, rekey), so the old dotenv path can never decrypt a blob into cwd. `cmd_rekey` and `cmd_list` recurse into `external/` explicitly (rekey MUST, or the blob is orphaned after rotation = data loss). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). +- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). - **Merge is pure bash, no `sed`/regex** (`merge_gradle_keys`): exact-string key comparison (avoids `beaconClerkPk` vs `beaconClerkPkTest` substring bug), value treated as opaque literal (survives `& \ /` in values). Updates a managed key in place at its first occurrence, collapses duplicates, appends new keys, preserves unrelated lines/comments/order. Continuation lines (trailing odd backslashes, tracked by `_trailing_bs_odd`) are never matched as keys. Atomic write: temp in the same dir → `chmod` to match (or `600` on create) → `mv`. Backs up to `.secrets-bak` before each merge. - **Properties separator parsing** (`_props_get`): key ends at the first `=`, `:`, or whitespace (after lstrip); handles `key=value`, `key = value`, `key:value`, `key value`; last definition wins. - **Security:** the write target comes from a committed file, so `_validate_external_target_path` locks it down — basename must be `gradle.properties`, must resolve inside `$HOME` (deepest-existing-ancestor resolved, symlink target/parent refused, `..` rejected). This blocks a malicious manifest from appending decrypted keys to `~/.gitconfig`/`~/.bashrc`. `_parse_secrets_files_manifest` rejects shell metacharacters/control chars in path and keys (path allows `[A-Za-z0-9/._~-]` only; keys allow `[A-Za-z0-9._-]` + space), mirrors the `.secrets-store` posture (no shell expansion, symlinked manifest skipped). @@ -119,7 +149,7 @@ Key routing rules: - QA/testing site behavior → invoke /qa or /qa-only - Code review/diff check → invoke /review - Visual polish → invoke /design-review -- Ship/deploy/PR → invoke /ship or /land-and-deploy +- Ship/deploy/PR → invoke /ship or /land-and-deploy (after reading `.ship-policy.json`; no AI adversarial/red-team/security-specialist review in this repo) - Save progress → invoke /context-save - Resume context → invoke /context-restore - Author a backlog-ready spec/issue → invoke /spec diff --git a/README.md b/README.md index f7e754d..421aad5 100644 --- a/README.md +++ b/README.md @@ -494,7 +494,14 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (174 tests across both files) +# Run the test suite (191 tests across both files) brew install bats-core bats test/ + +# Security regression subset — operator-local only (attack-payload fixtures). +# Required before ship; records sign-off in .gstack/security-signoff.json. +./test/run-security.sh ``` + +Hosted AI agents must not run the security script or perform red-team/adversarial +review on this repo — see `.ship-policy.json` and `CLAUDE.md`. diff --git a/test/run-security.sh b/test/run-security.sh new file mode 100755 index 0000000..e7168ab --- /dev/null +++ b/test/run-security.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Local-only security regression suite. Uses attack-payload fixtures on purpose. +# Do NOT ask hosted AI agents to run this script or to perform equivalent red-team review. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +if ! command -v bats >/dev/null 2>&1; then + echo "bats-core is required: brew install bats-core" >&2 + exit 1 +fi + +# Subset of the full suite: adversarial regressions + explicit SECURITY tests + +# closely related path/injection rails. Keeps the run focused and fast. +FILTER='SECURITY|F1:|F2:|F3:|F4:|F5:|command injection does not execute|outside HOME is refused|symlinked target is refused|shell metacharacters|command-substitution|symlinked .secrets-store is skipped|symlinked .secrets-files is ignored|symlinked .secrets.json is refused' + +echo "Security regression suite (operator-local only)" +echo "Repository policy: see .ship-policy.json" +echo "" + +bats --filter "$FILTER" test/ + +echo "" +echo "All filtered security regression tests passed." +echo "" + +if [ ! -t 0 ]; then + echo "Refusing non-interactive sign-off. Re-run in a terminal and complete operator certification." >&2 + exit 1 +fi + +read -r -p "Operator name: " OPERATOR +if [ -z "${OPERATOR//[[:space:]]/}" ]; then + echo "Operator name is required." >&2 + exit 1 +fi + +read -r -p "Type SIGNOFF to certify you ran this suite locally: " CONFIRM +if [ "$CONFIRM" != "SIGNOFF" ]; then + echo "Sign-off aborted (expected exactly SIGNOFF)." >&2 + exit 1 +fi + +SIGNOFF_DIR="$ROOT/.gstack" +mkdir -p "$SIGNOFF_DIR" +COMMIT="$(git rev-parse HEAD 2>/dev/null || echo unknown)" +TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +SIGNOFF_FILE="$SIGNOFF_DIR/security-signoff.json" + +# Escape operator name for JSON (minimal — names should not contain quotes). +OPERATOR_JSON="${OPERATOR//\\/\\\\}" +OPERATOR_JSON="${OPERATOR_JSON//\"/\\\"}" + +cat >"$SIGNOFF_FILE" < Date: Sun, 7 Jun 2026 13:50:51 -0700 Subject: [PATCH 11/70] chore: bump version and changelog (v0.4.0.0) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e6e27..d7f6ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.4.0.0] - 2026-06-07 + +### Added + +- **`.secrets.json` manifest (EGB-677 stage 1)** — a committed, project-root + manifest is now the source of truth for what syncs. List the env files you + want under `dotenv[]` (project-relative, nested paths and `@`-scoped + workspaces allowed; `..`, absolute, and symlink paths are rejected) and + out-of-project files under `external[]` (`properties` or `file`). The + manifest is shared across machines, so a teammate who clones the project + sees exactly what to pull. +- **`secrets add `** — declare an env file in the manifest without + pushing. Bootstraps `.secrets.json` on first use, dedupes, and writes a + stable canonical form. +- **Auto-add on push** — `secrets push` discovers new `.env*` / `.dev.vars` + files and adds them to the manifest (prints what it added and reminds you to + commit). Gated by `options.autoAdd` in the manifest (default on); + `push --frozen` syncs only declared files, and `push --dry-run` previews + what would change without writing anything. +- **Manifest-driven pull** — restores every declared file, recreating nested + directories as needed, with the same path-safety rail applied at restore + time so a malicious committed manifest can't write outside the project. An + empty manifest is a safe no-op. +- **Legacy `.secrets-files` absorb** — an existing `.secrets-files` is folded + into `.secrets.json` on first push (gradle-properties entries become + `properties`); on pull the legacy file is superseded with a warning. +- **Platform-aware install hints** — missing-dependency errors now print the + right install command for your platform (brew / apt-get / dnf). + +### Changed + +- `jq` is required only when a manifest is present or being written; + manifest-less projects keep working without `jq` (manifest features are + skipped with a notice). + +### Fixed + +- **Key rotation no longer orphans nested or external blobs.** `secrets rekey` + and `secrets list` now walk the entire project tree, so nested manifest + entries (`/.age`) and `external/` blobs are re-encrypted + and listed correctly. Previously a rekey could leave nested blobs encrypted + under the discarded old key, making them permanently undecryptable. +- Test assertions now fail correctly under system bash 3.2 (standalone + `[[ ]]` checks no longer pass silently). + ## [0.3.0.0] - 2026-06-07 ### Added diff --git a/VERSION b/VERSION index 1da00ae..9551b0d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0.0 +0.4.0.0 From a35bff8d4b214c99e3225dd7770ee6bda7e98885 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:53:05 -0700 Subject: [PATCH 12/70] docs: correct manifest.bats and total test counts for v0.4.0.0 CLAUDE.md: manifest.bats 58 -> 60 tests (actual @test count). README.md: total 191 -> 193 tests across both files. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 174bf07..262818d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,7 +74,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (58 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (60 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index 421aad5..770a86d 100644 --- a/README.md +++ b/README.md @@ -494,7 +494,7 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (191 tests across both files) +# Run the test suite (193 tests across both files) brew install bats-core bats test/ From 52528f2e0642fc858b4b4725621a5dd8392396de Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 14:55:42 -0700 Subject: [PATCH 13/70] =?UTF-8?q?feat:=20secrets=20verify=20=E2=80=94=20ma?= =?UTF-8?q?nifest=E2=86=94store=20consistency=20+=20decrypt=20integrity=20?= =?UTF-8?q?(EGB-698)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only integrity check, the safety net for the stage-2 store migration. Default mode (current project) cross-checks $PWD/.secrets.json against the store both ways — declared-but-missing blobs and orphaned blobs (no manifest entry) — and decrypt-tests every dotenv + external blob with the current key, streaming plaintext to /dev/null so nothing is ever written to disk. `verify --all` decrypt-tests every blob in every project (integrity only; the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (find -type f), the same walk rekey/list use, so nested manifest blobs are covered. Exits non-zero on any finding so it can gate `migrate --finalize` and CI. 12 bats tests (clean, nested+external, missing blob, decrypt failure, orphan, missing external, no-manifest die, symlink refusal, --all clean/corrupt/orphan, nested decrypt failure). Full suite 205/205. bash 3.2 clean. --- CLAUDE.md | 3 +- README.md | 2 + secrets | 158 +++++++++++++++++++++++++++++++++++++++++++++ test/manifest.bats | 129 ++++++++++++++++++++++++++++++++++++ 4 files changed, 291 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 262818d..296e205 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,12 +56,13 @@ skips security specialist + red team, and Step 11 skips adversarial review. ## Architecture -Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. +Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify. - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. +- Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` - Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`) diff --git a/README.md b/README.md index 770a86d..fc6c1c7 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,8 @@ secrets clear | `secrets list` | Show all projects that have stored secrets | | `secrets rm ` | Delete a project's secrets from the store | | `secrets rekey` | Generate a new encryption key and re-encrypt everything | +| `secrets verify` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob | +| `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | ### Automatic project detection diff --git a/secrets b/secrets index ba2adce..61ea874 100755 --- a/secrets +++ b/secrets @@ -1907,6 +1907,161 @@ cmd_which() { fi } +# Decrypt-test one blob with the current key. Plaintext is streamed to +# /dev/null and never written to disk (read-only contract). Returns 0 if the +# blob decrypts, non-zero otherwise. +_verify_blob_decrypts() { + age -d -i "$KEY_FILE" "$1" >/dev/null 2>&1 +} + +# `secrets verify --all` — store-wide decrypt sweep. Decrypt-tests every blob +# in every project. No manifest consistency check: the store carries only +# ciphertext (manifests live in each project's repo), so orphan/missing +# detection is impossible store-wide. This is the migration integrity gate. +_verify_all() { + local checked=0 failed=0 dir project f rel + for dir in "$SECRETS_DIR"/*/; do + [ -d "$dir" ] || continue + project=$(basename "$dir") + [[ "$project" == .* ]] && continue + # Recurse the whole project tree (top-level / nested / external) — never a + # non-recursive glob, or nested blobs would be silently skipped (the exact + # class of bug that orphaned nested blobs on rekey before v0.4.0.0). + while IFS= read -r f; do + [ -f "$f" ] || continue + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$f"; then + rel="${f#"$SECRETS_DIR"/}" + echo "FAIL: $rel does not decrypt with the current key." >&2 + failed=$((failed + 1)) + fi + done < <(find "$dir" -type f -name '*.age') + done + if [ "$checked" -eq 0 ]; then + info "verify --all: store is empty — nothing to check." + return 0 + fi + if [ "$failed" -gt 0 ]; then + echo "verify --all: $failed of $checked blob(s) failed to decrypt." >&2 + return 1 + fi + info "verify --all: OK — all $checked blob(s) decrypt with the current key (integrity only; run 'secrets verify' in a project for manifest consistency)." +} + +# `secrets verify [project]` — current-project consistency + decrypt check. +# Cross-checks the committed .secrets.json against the store both ways +# (declared-but-missing blobs, orphaned blobs) and decrypt-tests every blob. +_verify_project() { + local explicit_project="$1" + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ ! -e "$manifest" ]; then + die "No $SECRETS_JSON_NAME in $PWD. + 'secrets verify' checks a project's manifest against the store. Either cd + into a project that has a $SECRETS_JSON_NAME, or run 'secrets verify --all' + for a store-wide decrypt sweep." + fi + _check_manifest_file "$manifest" # fatal on symlink / malformed / version + + local project + project=$(derive_project_name "$explicit_project") + local pdir="$SECRETS_DIR/$project" + + local findings=0 checked=0 + # `expected` accumulates the store-relative blob paths the manifest implies, + # newline-framed (leading + trailing \n per entry) so the orphan walk can + # test membership. bash 3.2 has no associative arrays — this string-set + + # case match mirrors the `seen` pattern in _external_entries_for_push. + local expected=$'\n' + + # ── dotenv entries: rail + missing-blob + decrypt ── + local rel blob + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then + echo "FINDING: unsafe dotenv path in $SECRETS_JSON_NAME: '$rel' (will be refused)." >&2 + findings=$((findings + 1)) + continue + fi + expected="$expected$rel.age"$'\n' + blob="$pdir/$rel.age" + if [ ! -f "$blob" ]; then + echo "FINDING: '$rel' is declared in $SECRETS_JSON_NAME but has no blob in the store ($project/$rel.age missing). Run 'secrets push'." >&2 + findings=$((findings + 1)) + continue + fi + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$blob"; then + echo "FINDING: blob for '$rel' ($project/$rel.age) does not decrypt with the current key." >&2 + findings=$((findings + 1)) + fi + done < <(jq -r '.dotenv // [] | .[]' "$manifest") + + # ── external entries: missing-blob + decrypt ── + local etype epath ekeys slug erel eblob + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + slug=$(_secrets_files_slug "$epath") + erel="external/$slug.$etype.age" + expected="$expected$erel"$'\n' + eblob="$pdir/$erel" + if [ ! -f "$eblob" ]; then + echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 + findings=$((findings + 1)) + continue + fi + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$eblob"; then + echo "FINDING: external blob for '$epath' ($project/$erel) does not decrypt with the current key." >&2 + findings=$((findings + 1)) + fi + done < <(_json_external_entries "$manifest") + + # ── orphan detection: any stored blob the manifest doesn't account for ── + if [ -d "$pdir" ]; then + local f frel + while IFS= read -r f; do + [ -f "$f" ] || continue + frel="${f#"$pdir"/}" + case "$expected" in + *$'\n'"$frel"$'\n'*) : ;; # accounted for by a manifest entry + *) + echo "FINDING: orphan blob $project/$frel has no entry in $SECRETS_JSON_NAME." >&2 + findings=$((findings + 1)) + ;; + esac + done < <(find "$pdir" -type f -name '*.age') + fi + + if [ "$findings" -gt 0 ]; then + echo "verify: $findings finding(s) for project '$project'." >&2 + return 1 + fi + info "verify: OK — $checked blob(s) verified for '$project' (manifest and store agree; all decrypt)." +} + +# `secrets verify [--all] [project]` — read-only integrity check. Exits +# non-zero on any finding so it can gate `migrate --finalize` and CI. +cmd_verify() { + resolve_store + check_initialized + check_key + + local all=false explicit_project="" + while [ $# -gt 0 ]; do + case "$1" in + --all) all=true; shift ;; + -*) die "Unknown verify flag: $1. Usage: secrets verify [--all] [project]" ;; + *) explicit_project="$1"; shift ;; + esac + done + + if [ "$all" = true ]; then + _verify_all + else + _verify_project "$explicit_project" + fi +} + cmd_help() { cat << 'EOF' secrets — encrypted secret file sync between machines @@ -1926,6 +2081,8 @@ Usage: secrets list List all projects and their secret files secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key + secrets verify [project] Check the manifest against the store + decrypt every blob + secrets verify --all Decrypt-test every blob in every project (integrity gate) secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` @@ -2094,6 +2251,7 @@ case "${1:-help}" in list) cmd_list ;; rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; + verify) shift; cmd_verify "$@" ;; which|where|status) cmd_which ;; help|--help|-h) cmd_help ;; *) die "Unknown command: $1. Run 'secrets help' for usage." ;; diff --git a/test/manifest.bats b/test/manifest.bats index 4459a6c..2a226d1 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -731,3 +731,132 @@ m_nojq_path() { [ "$status" -eq 0 ] [[ "$output" == *"packages/web/.env.development"* ]] || false } + +# ─── K: secrets verify — manifest↔store consistency + decrypt integrity (EGB-698) ─ + +@test "verify: clean pushed project reports OK and exits 0" { + init_with_remote + create_project_dir verifyok + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" verify + [ "$status" -eq 0 ] + [[ "$output" == *"OK"* ]] || false +} + +@test "verify: nested + external entries all pass" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir verifymix + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" verify + [ "$status" -eq 0 ] +} + +@test "verify: declared-but-missing blob is a finding (exit 1)" { + init_with_remote + create_project_dir verifymiss + "$SECRETS_BIN" push >/dev/null 2>&1 + rm "$SECRETS_DIR/verifymiss/.env.staging.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".env.staging"* ]] || false +} + +@test "verify: a blob that fails to decrypt is a finding (exit 1)" { + init_with_remote + create_project_dir verifycorrupt + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'not-a-valid-age-blob' > "$SECRETS_DIR/verifycorrupt/.env.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".env"* ]] || false +} + +@test "verify: orphan blob (no manifest entry) is a finding (exit 1)" { + init_with_remote + create_project_dir verifyorphan + "$SECRETS_BIN" push >/dev/null 2>&1 + # A valid, decryptable blob with no manifest entry — pure consistency miss. + cp "$SECRETS_DIR/verifyorphan/.env.age" "$SECRETS_DIR/verifyorphan/.stray.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"stray"* ]] || false +} + +@test "verify: missing external blob is a finding (exit 1)" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir verifyextmiss + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + rm "$SECRETS_DIR/verifyextmiss/external/"*.age + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] +} + +@test "verify: no manifest in cwd dies with a directed message" { + init_with_remote + local dir="$WORK_DIR/verifynomani"; mkdir -p "$dir"; cd "$dir" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *"--all"* ]] || false +} + +@test "verify: symlinked manifest is refused" { + init_with_remote + create_project_dir verifysymlink + "$SECRETS_BIN" push >/dev/null 2>&1 + rm .secrets.json + ln -s /etc/hosts .secrets.json + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"symlink"* ]] || false +} + +@test "verify --all: clean store passes (decrypt-only)" { + init_with_remote + create_project_dir verifyall1 + "$SECRETS_BIN" push >/dev/null 2>&1 + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 0 ] +} + +@test "verify --all: a corrupted blob anywhere fails (exit 1)" { + init_with_remote + create_project_dir verifyall2 + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'garbage' > "$SECRETS_DIR/verifyall2/.env.age" + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 1 ] +} + +@test "verify --all: an orphan that decrypts is NOT flagged (no consistency check)" { + init_with_remote + create_project_dir verifyall3 + "$SECRETS_BIN" push >/dev/null 2>&1 + # Orphan blob that decrypts fine — default mode flags it, --all does not. + cp "$SECRETS_DIR/verifyall3/.env.age" "$SECRETS_DIR/verifyall3/.stray.age" + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 0 ] +} + +@test "verify: nested blob that fails to decrypt is caught (rekey-orphan guard)" { + init_with_remote + create_project_dir verifynested + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'broken' > "$SECRETS_DIR/verifynested/packages/web/.env.development.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"packages/web/.env.development"* ]] || false +} From 33aad4f89a173dfe72328089d18ddd2110d5f91c Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 15:14:54 -0700 Subject: [PATCH 14/70] test: coverage for verify gap branches (external decrypt-fail, unsafe path, unknown flag, malformed manifest, empty --all) Coverage audit found 5 untested branches in cmd_verify (all single-test fills, no logic defects): external blob decrypt-failure (only the missing case was covered), the rail-skip finding for an unsafe dotenv path in the manifest, the unknown-flag die, a malformed manifest through the verify entry point, and the empty-store 'verify --all' no-op. Full suite 210/210. --- test/manifest.bats | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/manifest.bats b/test/manifest.bats index 2a226d1..3719e8e 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -860,3 +860,53 @@ m_nojq_path() { [ "$status" -eq 1 ] [[ "$output" == *"packages/web/.env.development"* ]] || false } + +@test "verify: external blob that fails to decrypt is a finding (exit 1)" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir verifyextcorrupt + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'garbage' > "$SECRETS_DIR/verifyextcorrupt/external/"*.age + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"external"* ]] || false +} + +@test "verify: an unsafe dotenv path in the manifest is a finding (exit 1)" { + init_with_remote + create_project_dir verifyunsafe + "$SECRETS_BIN" push >/dev/null 2>&1 + # Hand-edit the committed manifest to declare a traversal path the rail refuses. + jq '.dotenv += ["../evil"]' .secrets.json > .secrets.json.tmp && mv .secrets.json.tmp .secrets.json + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"unsafe"* ]] || false +} + +@test "verify: unknown flag dies with usage" { + init_with_remote + create_project_dir verifyflag + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" verify --bogus + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown verify flag"* ]] || false +} + +@test "verify: malformed manifest is refused" { + init_with_remote + create_project_dir verifymalformed + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'not json{' > .secrets.json + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"JSON"* ]] || false +} + +@test "verify --all: empty store reports nothing to check (exit 0)" { + init_with_remote + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 0 ] + [[ "$output" == *"empty"* ]] || false +} From 414c02b902efa110b986659a461b4826985ff0df Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 15:22:58 -0700 Subject: [PATCH 15/70] fix: pre-landing review fixes for verify (test assertions, ekeys discard, double-report, docs) Pre-landing review (0 critical, all informational) auto-fixes: - Tighten external-corrupt test to assert the decrypt-fail message, not any external finding (was *"external"*, now *"does not decrypt"*). - Pin the verified-count in the nested+external happy-path test so a silent under-count (exit 0 while skipping a blob) is caught. - Account for an unsafe dotenv entry in `expected` so a matching stray blob isn't double-reported as both unsafe and orphan. - Discard the unused external `keys` read field (read -r etype epath _). - Document the optional [project] positional in the README verify row. Deferred to EGB-701 (stage-2 dedup): the external blob-path literal and the find-walk overlap with cmd_rekey/cmd_list. Full suite 210/210. --- README.md | 2 +- secrets | 8 ++++++-- test/manifest.bats | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fc6c1c7..e79fc15 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ secrets clear | `secrets list` | Show all projects that have stored secrets | | `secrets rm ` | Delete a project's secrets from the store | | `secrets rekey` | Generate a new encryption key and re-encrypt everything | -| `secrets verify` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob | +| `secrets verify [project]` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob. `[project]` overrides the store directory name; the manifest is still read from the current directory | | `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | ### Automatic project detection diff --git a/secrets b/secrets index 61ea874..5ed68a5 100755 --- a/secrets +++ b/secrets @@ -1980,6 +1980,9 @@ _verify_project() { if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then echo "FINDING: unsafe dotenv path in $SECRETS_JSON_NAME: '$rel' (will be refused)." >&2 findings=$((findings + 1)) + # Still account for it so a matching stray blob isn't ALSO flagged as an + # orphan (one bad entry → one finding, not two). + expected="$expected$rel.age"$'\n' continue fi expected="$expected$rel.age"$'\n' @@ -1997,8 +2000,9 @@ _verify_project() { done < <(jq -r '.dotenv // [] | .[]' "$manifest") # ── external entries: missing-blob + decrypt ── - local etype epath ekeys slug erel eblob - while IFS=$'\t' read -r etype epath ekeys; do + # verify only needs type + path to locate the blob; keys are irrelevant here. + local etype epath slug erel eblob + while IFS=$'\t' read -r etype epath _; do [ -n "$etype" ] || continue slug=$(_secrets_files_slug "$epath") erel="external/$slug.$etype.age" diff --git a/test/manifest.bats b/test/manifest.bats index 3719e8e..9df5bfa 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -754,6 +754,9 @@ m_nojq_path() { "$SECRETS_BIN" push >/dev/null 2>&1 run "$SECRETS_BIN" verify [ "$status" -eq 0 ] + # Pin the count so a silent under-count (e.g. skipping the nested or external + # blob while still exiting 0) is caught: 3 dotenv + 1 external = 4. + [[ "$output" == *"4 blob(s) verified"* ]] || false } @test "verify: declared-but-missing blob is a finding (exit 1)" { @@ -870,7 +873,8 @@ m_nojq_path() { printf 'garbage' > "$SECRETS_DIR/verifyextcorrupt/external/"*.age run "$SECRETS_BIN" verify [ "$status" -eq 1 ] - [[ "$output" == *"external"* ]] || false + # Pin the decrypt-fail branch specifically, not just any external finding. + [[ "$output" == *"does not decrypt"* ]] || false } @test "verify: an unsafe dotenv path in the manifest is a finding (exit 1)" { From d170fb03db1a0bbe1cd2cdaf6e2c2b0cd1903aeb Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 15:29:01 -0700 Subject: [PATCH 16/70] chore: bump version and changelog (v0.5.0.0) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 17 +++++++++++++++++ VERSION | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7f6ecf..bc91609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.5.0.0] - 2026-06-07 + +### Added + +- **`secrets verify` (EGB-698)** — a read-only integrity check. Run it in a + project to cross-check the committed `.secrets.json` against the store both + ways (entries declared but missing from the store, and stored blobs with no + manifest entry) and decrypt-test every blob with your current key. Catches a + partially-synced store, a stale key, or a manifest that has drifted from the + store. Plaintext is streamed to `/dev/null` and never written to disk. +- **`secrets verify --all`** — decrypt-tests every blob in every project in the + store: a fast store-wide integrity sweep. (The store carries no manifests, so + `--all` checks decryptability only, not manifest consistency.) +- Both modes recurse the whole project tree, so nested entries and external + files are covered. `secrets verify` exits non-zero on any problem, so it can + gate CI or a future store migration. + ## [0.4.0.0] - 2026-06-07 ### Added diff --git a/VERSION b/VERSION index 9551b0d..eddcc3f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0.0 +0.5.0.0 From e33bc272d2c796efaa203906733e80d07558292a Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 15:31:30 -0700 Subject: [PATCH 17/70] docs: surface secrets verify in troubleshooting, refresh test counts (v0.5.0.0) Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 2 +- README.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 296e205..b6c380f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (60 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (77 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index e79fc15..b92f7a5 100644 --- a/README.md +++ b/README.md @@ -493,10 +493,12 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` **"'jq' is not installed"** — Manifest features need `jq`. The error prints the install command for your platform. Manifest-less projects work without it. +**Not sure the store is intact?** — Run `secrets verify` in a project to check its `.secrets.json` against the store (declared-but-missing blobs and orphaned blobs) and decrypt-test every blob with your current key. Use `secrets verify --all` for a store-wide decrypt sweep across every project. It's read-only — plaintext is streamed to `/dev/null`, never written to disk — and exits non-zero if anything is wrong, so it's safe to run in CI. + ## Development ```bash -# Run the test suite (193 tests across both files) +# Run the test suite (210 tests across both files) brew install bats-core bats test/ From e2ad661da5e306a4a1e882a211b26fd43423d00e Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:07:31 -0700 Subject: [PATCH 18/70] feat: store-format-v2 self-describing migration (EGB-703, folds in EGB-700) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 2 of EGB-677. Makes the store self-describing and unifies the legacy external `properties` blob suffix, via a copy-forward migration that never destroys data until an explicit, gated finalize. Scope decision (see eureka): the EGB-677 CEO plan's "flatten dotenv blobs to basename" was dropped as LOSSY — it discards the restore relpath that makes the store self-describing and adds basename collisions. Engineering analysis (4 parallel design agents) showed the store is already relpath-self-describing; the only real v1→v2 delta is the `properties` suffix. This implements the minimal, safe v2 that achieves the epic's self-describing goal. What's added: - `.secrets-format` marker (committed, one line `2`). Absence ⇒ v1 (every pre-EGB-703 store). `_store_format()` reads it; `init` stamps fresh stores born-v2. `secrets which` prints `format: vN` (EGB-700 folded in). - `_external_blob_suffix(type)` — single source of truth for the external suffix (v2: gradle-properties → properties; file unchanged). push/pull/verify all route through it, so v1 and v2 stores never disagree on blob location. - `secrets migrate` — per-project copy-forward (writes `.properties.age` twins beside v1 blobs; idempotent; needs the project manifest), `--dry-run` (reports old→new, writes nothing), `--finalize` (store-wide, the only destructive step: gates on `verify --all` green + every v1 blob twinned, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation). rekey and verify --all stay format-agnostic (recursive find walk) — no change. 21 new bats tests (test/migrate.bats): marker/born-v2, format-aware suffix, v1 back-compat, dry-run, copy-forward idempotency, no-manifest die, finalize gates (verify-not-green refusal, untwinned refusal, recovery tag, confirmation), and full v1→window→finalize round-trip. Updated 4 existing tests for the born-v2 suffix. Full suite 231/231, bash 3.2 clean. --- CLAUDE.md | 5 +- README.md | 2 + secrets | 212 ++++++++++++++++++++++++++++++++++- test/manifest.bats | 4 +- test/migrate.bats | 270 +++++++++++++++++++++++++++++++++++++++++++++ test/secrets.bats | 4 +- 6 files changed, 487 insertions(+), 10 deletions(-) create mode 100644 test/migrate.bats diff --git a/CLAUDE.md b/CLAUDE.md index b6c380f..e953300 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,12 +56,13 @@ skips security specialist + red team, and Step 11 skips adversarial review. ## Architecture -Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify. +Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify, migrate. - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) -- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. +- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints `format: vN`. **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, writes `.properties.age` twins beside the v1 blobs; needs the project manifest to know which externals are `properties`; idempotent) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` diff --git a/README.md b/README.md index b92f7a5..354e23c 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,8 @@ secrets clear | `secrets rekey` | Generate a new encryption key and re-encrypt everything | | `secrets verify [project]` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob. `[project]` overrides the store directory name; the manifest is still read from the current directory | | `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | +| `secrets migrate [--dry-run]` | Copy-forward this project's encrypted blobs to store format v2 (non-destructive; `--dry-run` previews) | +| `secrets migrate --finalize` | Drop the old v1 blobs and mark the store v2 — runs once, store-wide, after `verify` is green and every machine is upgraded | ### Automatic project detection diff --git a/secrets b/secrets index 5ed68a5..600a25e 100755 --- a/secrets +++ b/secrets @@ -517,6 +517,39 @@ _secrets_files_slug() { printf '%s-%s' "$clean" "$sum" } +# ─── Store format (EGB-703) ─────────────────────────────────────────── +# +# The store self-describes its format via a committed one-line file +# `$SECRETS_DIR/.secrets-format` containing `2`. Absence (or any non-`2` +# content) means format v1 — the legacy default for every store that +# predates EGB-703. `init` stamps a fresh store v2 (born-v2); `migrate +# --finalize` stamps a migrated store v2. Requires resolve_store to have +# run (SECRETS_DIR set). +STORE_FORMAT_FILE_NAME=".secrets-format" +_store_format() { + local f="$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" v + if [ -f "$f" ]; then + v=$(head -1 "$f" 2>/dev/null | tr -dc '0-9') + [ "$v" = "2" ] && { echo 2; return; } + fi + echo 1 +} + +# The on-disk blob suffix for an external entry, format-aware. v2 unifies +# the legacy `gradle-properties` suffix to `properties` (matching the JSON +# manifest `type`); `file` is unchanged in both formats. The slug + this +# suffix + `.age` is the external blob name. This is the single source of +# truth for the suffix — push, pull, verify all route through it so a v1 +# and a v2 store can never disagree on where a blob lives. +_external_blob_suffix() { + local mtype="$1" + if [ "$mtype" = "gradle-properties" ] && [ "$(_store_format)" = "2" ]; then + echo "properties" + else + echo "$mtype" + fi +} + # Merge managed key=value lines (from $2) into target file $1, preserving # all unrelated lines/comments/order. Updates a managed key in place (first # occurrence), collapses duplicates, appends new keys. Atomic + mode-safe. @@ -615,7 +648,7 @@ push_external_files() { # EGB-652: whole-file sync — encrypt the file verbatim (binary-safe). mkdir -p "$SECRETS_DIR/$project/external" local fslug; fslug=$(_secrets_files_slug "$mpath") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.file.age" "$expanded" + age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.$(_external_blob_suffix file).age" "$expanded" info "Encrypted file $mpath" pushed=$((pushed + 1)) continue @@ -644,7 +677,7 @@ push_external_files() { fi mkdir -p "$SECRETS_DIR/$project/external" local slug; slug=$(_secrets_files_slug "$mpath") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$mtype.age" "$tmp" + age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" "$tmp" rm -f "$tmp" info "Extracted $found key(s) from $mpath" pushed=$((pushed + 1)) @@ -673,7 +706,7 @@ pull_external_files() { continue fi local slug; slug=$(_secrets_files_slug "$mpath") - local blob="$SECRETS_DIR/$project/external/$slug.$mtype.age" + local blob="$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" if [ ! -f "$blob" ]; then echo "WARNING: $SECRETS_FILES_NAME names '$mpath' but no encrypted data exists in the store yet. Run 'secrets push' on a machine that has these keys. Skipping." >&2 continue @@ -1126,6 +1159,11 @@ Your key file has been left untouched." # Write .gitignore write_store_gitignore + # Stamp the store format (EGB-703): a fresh store is born v2 — it has no + # v1 blobs, so it is already in v2 shape. The marker is a committed, + # non-secret metadata file (NOT gitignored); the first push stages it. + printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" + # Install pre-commit hook mkdir -p "$SECRETS_DIR/.git/hooks" install_hook @@ -1866,6 +1904,9 @@ cmd_which() { resolve_store echo "store: $SECRETS_DIR" echo "source: $STORE_SOURCE" + # EGB-700: surface the store format so users can tell v1 from v2 during the + # migration window. A v1 store is a legacy store with no format marker. + echo "format: v$(_store_format)" # v2 manifest (.secrets.json): validate and summarize. Validation here # is deliberately fatal (symlink / malformed / unsupported version) so @@ -2005,7 +2046,7 @@ _verify_project() { while IFS=$'\t' read -r etype epath _; do [ -n "$etype" ] || continue slug=$(_secrets_files_slug "$epath") - erel="external/$slug.$etype.age" + erel="external/$slug.$(_external_blob_suffix "$etype").age" expected="$expected$erel"$'\n' eblob="$pdir/$erel" if [ ! -f "$eblob" ]; then @@ -2066,6 +2107,166 @@ cmd_verify() { fi } +# ─── Store-format-v2 migration (EGB-703) ────────────────────────────── +# +# v2 renames the legacy `properties` blob suffix (.gradle-properties.age → +# .properties.age) and marks the store self-describing via .secrets-format. +# Migration is copy-forward and non-destructive until --finalize: +# secrets migrate --dry-run # per project: report old→new, write nothing +# secrets migrate # per project: write v2 twins beside v1 blobs +# secrets migrate --finalize # store-wide: verify, drop v1, stamp v2 +# Per-project (needs the project manifest to know which externals are +# `properties`); finalize is store-wide. Mirrors verify's project/--all split. + +# Copy-forward (or dry-run preview) for the current project. Reads +# $PWD/.secrets.json; only `properties` external blobs rename in v2. +_migrate_project() { + local dry_run="$1" + if [ "$(_store_format)" = "2" ]; then + info "Store is already format v2 — nothing to migrate." + return 0 + fi + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ ! -e "$manifest" ]; then + die "No $SECRETS_JSON_NAME in $PWD. + 'secrets migrate' copy-forwards a project's v1 blobs to their v2 names and + reads the project manifest to do so. cd into a project that has a manifest, + then run 'secrets migrate'. (Store-wide 'secrets migrate --finalize' comes + after every project is migrated.)" + fi + _check_manifest_file "$manifest" + local project; project=$(derive_project_name "") + local pdir="$SECRETS_DIR/$project" + + local moved=0 already=0 would=0 etype epath slug old new + while IFS=$'\t' read -r etype epath _; do + [ -n "$etype" ] || continue + # Only `properties` blobs change name in v2; dotenv and `file` are already + # in their v2 shape and never move. + [ "$etype" = "gradle-properties" ] || continue + slug=$(_secrets_files_slug "$epath") + old="$pdir/external/$slug.gradle-properties.age" + new="$pdir/external/$slug.properties.age" + [ -f "$old" ] || continue # nothing pushed yet (or already dropped) + if [ -f "$new" ]; then # idempotent: twin already exists + already=$((already + 1)) + continue + fi + if [ "$dry_run" = true ]; then + echo "would migrate: $project/external/$slug.gradle-properties.age -> $slug.properties.age" + would=$((would + 1)) + else + cp "$old" "$new" + moved=$((moved + 1)) + fi + done < <(_json_external_entries "$manifest") + + if [ "$dry_run" = true ]; then + echo "migrate --dry-run: $would blob(s) would be copy-forwarded for '$project' (writes nothing); $already already present. v1 blobs are kept until 'secrets migrate --finalize'." + return 0 + fi + if [ "$moved" -eq 0 ] && [ "$already" -eq 0 ]; then + info "Nothing to migrate for '$project' (no v1 properties blobs)." + return 0 + fi + ensure_store_protections + git -C "$SECRETS_DIR" add -A + git -C "$SECRETS_DIR" commit -m "migrate: copy-forward v2 twins for $project" >/dev/null 2>&1 || true + info "Copy-forward for '$project': $moved new v2 twin(s), $already already present. v1 blobs kept (non-destructive). Run 'secrets migrate --finalize' once every project is migrated and every machine is upgraded." +} + +# Store-wide finalize: the only destructive step. Refuses unless verify --all +# is green and every v1 properties blob has a v2 twin. Cuts a recovery tag, +# stamps the marker, then drops v1 blobs. +_migrate_finalize() { + local force="$1" + check_key + if [ "$(_store_format)" = "2" ]; then + info "Store is already format v2 — nothing to finalize." + return 0 + fi + + # (gate 1) every blob must decrypt with the current key. + info "Verifying every blob decrypts before finalizing..." + if ! _verify_all >/dev/null 2>&1; then + die "Refusing to finalize: 'secrets verify --all' is not green — a blob does not decrypt. Run 'secrets verify --all' to see which, fix it, then re-run --finalize." + fi + + # (gate 2) every v1 properties blob must have a v2 twin (project migrated). + local untwinned="" f new v1count=0 + while IFS= read -r f; do + [ -f "$f" ] || continue + v1count=$((v1count + 1)) + new="${f%.gradle-properties.age}.properties.age" + [ -f "$new" ] || untwinned="$untwinned ${f#"$SECRETS_DIR"/}"$'\n' + done < <(find "$SECRETS_DIR" -type f -name '*.gradle-properties.age') + if [ -n "$untwinned" ]; then + die "Refusing to finalize: these v1 properties blobs have no v2 twin (their project was not migrated): +$untwinned cd into each project and run 'secrets migrate', then re-run 'secrets migrate --finalize'." + fi + if [ "$v1count" -eq 0 ]; then + # No v1 blobs at all — just stamp the marker (dotenv/file-only store). + printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" + ensure_store_protections + git -C "$SECRETS_DIR" add -A + git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2" >/dev/null 2>&1 || true + git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true + info "Store finalized to format v2 (no v1 blobs to drop)." + return 0 + fi + + # (gate 3) operator confirms every machine is upgraded. + if [ "$force" != true ]; then + echo "Finalize will drop $v1count v1 blob(s) and stamp the store format v2." + echo "Any machine still running a pre-v2 'secrets' will stop seeing 'properties'" + echo "externals from this store until it upgrades (git pull in the secrets repo)." + printf "Confirm every machine is upgraded? Type 'yes': " + local reply="" + read -r reply < /dev/tty 2>/dev/null || read -r reply || true + [ "$reply" = "yes" ] || die "Finalize aborted — no confirmation." + fi + + # Recovery tag BEFORE any mutation: points at the pre-finalize commit (v1 + # blobs intact, no marker), so `git checkout ` fully restores v1. + local tag="pre-v2-migrate-$(git -C "$SECRETS_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" + git -C "$SECRETS_DIR" tag "$tag" >/dev/null 2>&1 || true + + # Stamp the marker FIRST, then drop v1 blobs. If finalize crashes between + # the two, the store reads as v2 and the (verified) v2 twins serve every + # upgraded client; leftover v1 blobs are harmless orphans a re-run cleans. + printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" + while IFS= read -r f; do + [ -f "$f" ] || continue + rm -f "$f" + done < <(find "$SECRETS_DIR" -type f -name '*.gradle-properties.age') + + ensure_store_protections + git -C "$SECRETS_DIR" add -A + git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2 (drop $v1count v1 blob(s))" >/dev/null 2>&1 || true + git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true + info "Store finalized to format v2. Dropped $v1count v1 blob(s). Recovery tag in the store: $tag" +} + +cmd_migrate() { + resolve_store + check_initialized + local dry_run=false finalize=false force=false + while [ $# -gt 0 ]; do + case "$1" in + --dry-run) dry_run=true; shift ;; + --finalize) finalize=true; shift ;; + --yes|--force) force=true; shift ;; + -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --finalize] [--yes]" ;; + *) die "migrate takes no project argument. Run it from inside a project (copy-forward) or use --finalize (store-wide)." ;; + esac + done + if [ "$finalize" = true ]; then + _migrate_finalize "$force" + else + _migrate_project "$dry_run" + fi +} + cmd_help() { cat << 'EOF' secrets — encrypted secret file sync between machines @@ -2087,6 +2288,8 @@ Usage: secrets rekey Re-encrypt all secrets with a new key secrets verify [project] Check the manifest against the store + decrypt every blob secrets verify --all Decrypt-test every blob in every project (integrity gate) + secrets migrate [--dry-run] Copy-forward this project's blobs to store format v2 + secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` @@ -2256,6 +2459,7 @@ case "${1:-help}" in rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; verify) shift; cmd_verify "$@" ;; + migrate) shift; cmd_migrate "$@" ;; which|where|status) cmd_which ;; help|--help|-h) cmd_help ;; *) die "Unknown command: $1. Run 'secrets help' for usage." ;; diff --git a/test/manifest.bats b/test/manifest.bats index 9df5bfa..37564c7 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -269,7 +269,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > run jq -r '.external[] | select(.type == "file") | .path' .secrets.json [ "$output" = "~/keystores/upload.keystore" ] # stage 1: blob naming stays legacy-compatible - run bash -c "ls $SECRETS_DIR/absorbproj/external/*.gradle-properties.age" + run bash -c "ls $SECRETS_DIR/absorbproj/external/*.properties.age" [ "$status" -eq 0 ] } @@ -296,7 +296,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > run "$SECRETS_BIN" push jsonextproj [ "$status" -eq 0 ] [[ "$output" == *"Extracted 1 key"* ]] || false - run bash -c "ls $SECRETS_DIR/jsonextproj/external/*.gradle-properties.age" + run bash -c "ls $SECRETS_DIR/jsonextproj/external/*.properties.age" [ "$status" -eq 0 ] } diff --git a/test/migrate.bats b/test/migrate.bats new file mode 100644 index 0000000..b7559a6 --- /dev/null +++ b/test/migrate.bats @@ -0,0 +1,270 @@ +#!/usr/bin/env bats +# EGB-703 store-format-v2: marker, format-aware suffix, migrate (dry-run / +# copy-forward / finalize). bash 3.2: every standalone [[ ]] ends with || false. + +load test_helper + +# A v1 (legacy) store: born-v2 init, then strip the marker so it reads as v1 +# and pushes write the legacy .gradle-properties.age suffix. +make_v1_store() { + init_with_remote + rm -f "$SECRETS_DIR/.secrets-format" +} +m_gradle_src() { mkdir -p "$HOME/.gradle"; printf '%s' "$1" > "$HOME/.gradle/gradle.properties"; } +m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > "$HOME/keystores/upload.keystore"; } + +# ─── Format marker + format-aware suffix (increment 1) ──────────────── + +@test "init stamps the store format marker as v2 (born-v2)" { + init_with_remote + [ -f "$SECRETS_DIR/.secrets-format" ] + [ "$(cat "$SECRETS_DIR/.secrets-format")" = "2" ] +} + +@test "which prints format v2 for a born-v2 store" { + init_with_remote + create_project_dir whichv2 + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"format: v2"* ]] || false +} + +@test "which prints format v1 for a markerless (legacy) store" { + make_v1_store + create_project_dir whichv1 + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"format: v1"* ]] || false +} + +@test "push on a born-v2 store writes the properties blob as .properties.age" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir v2push + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push v2push >/dev/null 2>&1 + run bash -c "ls $SECRETS_DIR/v2push/external/*.properties.age" + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/v2push/external/*.gradle-properties.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "push on a v1 store still writes .gradle-properties.age (back-compat)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir v1push + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push v1push >/dev/null 2>&1 + run bash -c "ls $SECRETS_DIR/v1push/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "the format marker is committed, not gitignored" { + init_with_remote + create_project_dir markercommit + "$SECRETS_BIN" push markercommit >/dev/null 2>&1 + run bash -c "git -C $SECRETS_DIR ls-files | grep -qx .secrets-format" + [ "$status" -eq 0 ] +} + +# ─── migrate --dry-run / copy-forward (increment 2) ─────────────────── + +@test "migrate --dry-run reports the rename and writes nothing" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir dryproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push dryproj >/dev/null 2>&1 + run "$SECRETS_BIN" migrate --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"would migrate"* ]] || false + # nothing written + run bash -c "ls $SECRETS_DIR/dryproj/external/*.properties.age 2>/dev/null" + [ "$status" -ne 0 ] + # marker still absent (store still v1) + [ ! -f "$SECRETS_DIR/.secrets-format" ] +} + +@test "migrate --dry-run on a dotenv-only project reports nothing to migrate" { + make_v1_store + create_project_dir dotenvonly + "$SECRETS_BIN" push dotenvonly >/dev/null 2>&1 + run "$SECRETS_BIN" migrate --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"0 blob(s) would be copy-forwarded"* ]] || false +} + +@test "migrate copy-forward creates the v2 twin and keeps the v1 blob (byte-identical)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir cfproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push cfproj >/dev/null 2>&1 + local old; old=$(ls "$SECRETS_DIR/cfproj/external/"*.gradle-properties.age) + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + local new; new=$(ls "$SECRETS_DIR/cfproj/external/"*.properties.age) + [ -f "$old" ] # v1 kept (non-destructive) + [ -f "$new" ] # v2 twin written + cmp -s "$old" "$new" # byte-identical ciphertext copy +} + +@test "migrate copy-forward is idempotent" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir idemproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push idemproj >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + [[ "$output" == *"1 already present"* ]] || false + run bash -c "ls $SECRETS_DIR/idemproj/external/*.properties.age | wc -l | tr -d ' '" + [ "$output" = "1" ] +} + +@test "migrate with no manifest in cwd dies with a directed message" { + make_v1_store + local dir="$WORK_DIR/nomanifest"; mkdir -p "$dir"; cd "$dir" + run "$SECRETS_BIN" migrate + [ "$status" -eq 1 ] + [[ "$output" == *".secrets.json"* ]] || false +} + +@test "migrate on an already-v2 store is a no-op" { + init_with_remote + create_project_dir alreadyv2 + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + [[ "$output" == *"already format v2"* ]] || false +} + +@test "migrate unknown flag dies with usage" { + init_with_remote + create_project_dir mgflag + run "$SECRETS_BIN" migrate --bogus + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown migrate flag"* ]] || false +} + +@test "migrate leaves dotenv and file blobs untouched" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + m_file_src + create_project_dir mixproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\nfile ~/keystores/upload.keystore\n' > .secrets-files + "$SECRETS_BIN" push mixproj >/dev/null 2>&1 + local envblob; envblob=$(ls "$SECRETS_DIR/mixproj/".env.age) + local fileblob; fileblob=$(ls "$SECRETS_DIR/mixproj/external/"*.file.age) + local envsum; envsum=$(cksum "$envblob") + local filesum; filesum=$(cksum "$fileblob") + "$SECRETS_BIN" migrate >/dev/null 2>&1 + [ "$(cksum "$envblob")" = "$envsum" ] # dotenv blob unchanged + [ "$(cksum "$fileblob")" = "$filesum" ] # file blob unchanged +} + +# ─── migrate --finalize (increment 3) ───────────────────────────────── + +@test "finalize refuses when verify --all is not green" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir failverify + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push failverify >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + # corrupt the v2 twin so verify --all fails + printf 'garbage' > "$SECRETS_DIR/failverify/external/"*.properties.age + run "$SECRETS_BIN" migrate --finalize --yes + [ "$status" -eq 1 ] + [[ "$output" == *"not green"* ]] || false + # marker not stamped; v1 blob still present + [ ! -f "$SECRETS_DIR/.secrets-format" ] + run bash -c "ls $SECRETS_DIR/failverify/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "finalize refuses an un-twinned v1 blob (project not migrated)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir untwinned + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push untwinned >/dev/null 2>&1 + # do NOT migrate — leave the v1 blob with no twin + run "$SECRETS_BIN" migrate --finalize --yes + [ "$status" -eq 1 ] + [[ "$output" == *"no v2 twin"* ]] || false + run bash -c "ls $SECRETS_DIR/untwinned/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "finalize green path drops v1, keeps v2, stamps the marker" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir finproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push finproj >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + run "$SECRETS_BIN" migrate --finalize --yes + [ "$status" -eq 0 ] + [ "$(cat "$SECRETS_DIR/.secrets-format")" = "2" ] + run bash -c "ls $SECRETS_DIR/finproj/external/*.properties.age" + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/finproj/external/*.gradle-properties.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "finalize cuts a recovery tag before deleting v1 blobs" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir tagproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push tagproj >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + "$SECRETS_BIN" migrate --finalize --yes >/dev/null 2>&1 + local tag; tag=$(git -C "$SECRETS_DIR" tag | grep '^pre-v2-migrate-') + [ -n "$tag" ] + # the tagged commit still contains the v1 blob (tag cut before delete) + run bash -c "git -C $SECRETS_DIR ls-tree -r --name-only $tag | grep -q gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "finalize without --yes aborts when not confirmed" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir confproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push confproj >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + run bash -c "echo '' | $SECRETS_BIN migrate --finalize" + [ "$status" -eq 1 ] + [[ "$output" == *"aborted"* ]] || false + [ ! -f "$SECRETS_DIR/.secrets-format" ] +} + +@test "v1 client still reads during the migration window (after copy-forward, before finalize)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir windowproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push windowproj >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + # store is still v1 (markerless); pull uses the .gradle-properties.age blob + rm "$HOME/.gradle/gradle.properties" + run "$SECRETS_BIN" pull windowproj + [ "$status" -eq 0 ] + grep -q '^beaconClerkPkTest=pk_test_abc$' "$HOME/.gradle/gradle.properties" +} + +@test "post-finalize pull reads the v2 blob" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir postfin + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push postfin >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + "$SECRETS_BIN" migrate --finalize --yes >/dev/null 2>&1 + rm "$HOME/.gradle/gradle.properties" + run "$SECRETS_BIN" pull postfin + [ "$status" -eq 0 ] + grep -q '^beaconClerkPkTest=pk_test_abc$' "$HOME/.gradle/gradle.properties" +} diff --git a/test/secrets.bats b/test/secrets.bats index 5537a03..a75c0e6 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1081,7 +1081,7 @@ gradle_project() { run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] [[ "$output" == *"Extracted 2 key"* ]] || false - run bash -c "ls $SECRETS_DIR/gproj/external/*.gradle-properties.age" + run bash -c "ls $SECRETS_DIR/gproj/external/*.properties.age" [ "$status" -eq 0 ] } @@ -1368,7 +1368,7 @@ gradle_project() { run "$SECRETS_BIN" push -w [ "$status" -eq 0 ] # External blob pushed exactly once (not once per workspace) - run bash -c "ls $SECRETS_DIR/mono/external/*.gradle-properties.age 2>/dev/null | wc -l | tr -d ' '" + run bash -c "ls $SECRETS_DIR/mono/external/*.properties.age 2>/dev/null | wc -l | tr -d ' '" [ "$output" = "1" ] } From 2192b5a2df3078fba88b0d044903dfcc945d89f6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:18:16 -0700 Subject: [PATCH 19/70] fix: tighten _store_format parse + correct finalize crash-orphan comment (EGB-703 review) Self-review of the data-safety paths (verdict SHIP-SAFE) flagged two non-blocking nits, both fixed: - _store_format used `tr -dc '0-9'` which read garbage like "v2"/"x2x" as v2. Tightened to a strict exact match (modulo line endings) so only "2" reads as v2; anything else falls back to v1, the safe default. - The stamp-before-delete comment claimed a re-run "cleans" crash-orphaned v1 blobs; it doesn't (finalize early-returns once the store is v2). Corrected to note the orphans are harmless and `verify` flags them for manual removal. --- secrets | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/secrets b/secrets index 600a25e..2328fdc 100755 --- a/secrets +++ b/secrets @@ -529,7 +529,11 @@ STORE_FORMAT_FILE_NAME=".secrets-format" _store_format() { local f="$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" v if [ -f "$f" ]; then - v=$(head -1 "$f" 2>/dev/null | tr -dc '0-9') + # Strict exact match (modulo line endings): only a marker of exactly "2" + # reads as v2. Anything else (empty, "20", "v2", garbage) ⇒ v1 — the safe + # default, since misreading v2-as-v1 only triggers a harmless re-migrate + # while v1-as-v2 would mislocate blobs. + v=$(head -1 "$f" 2>/dev/null | tr -d '\r\n') [ "$v" = "2" ] && { echo 2; return; } fi echo 1 @@ -2233,7 +2237,9 @@ $untwinned cd into each project and run 'secrets migrate', then re-run 'secrets # Stamp the marker FIRST, then drop v1 blobs. If finalize crashes between # the two, the store reads as v2 and the (verified) v2 twins serve every - # upgraded client; leftover v1 blobs are harmless orphans a re-run cleans. + # upgraded client; any leftover v1 blobs are harmless orphans (referenced by + # no v2 suffix lookup) that `secrets verify` will flag and the operator can + # remove. A re-run of --finalize early-returns (store is already v2). printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" while IFS= read -r f; do [ -f "$f" ] || continue From 5521d978a3d098f66acabb028b720ef09d43ee2f Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:35:26 -0700 Subject: [PATCH 20/70] test: coverage for migrate gap branches (dotenv-only finalize, positional-arg die, already-v2 finalize, strict marker parse) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage audit found 4 untested branches (all single-test fills): finalize on a v1 dotenv/file-only store (v1count==0 stamp path — a real user scenario), the migrate positional-arg die, finalize on an already-v2 store, and the _store_format strict-parse defense (garbage marker reads as v1). Full suite 235/235. --- test/migrate.bats | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/migrate.bats b/test/migrate.bats index b7559a6..35102f6 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -268,3 +268,40 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -eq 0 ] grep -q '^beaconClerkPkTest=pk_test_abc$' "$HOME/.gradle/gradle.properties" } + +@test "finalize on a v1 dotenv-only store stamps the marker (no v1 blobs to drop)" { + make_v1_store + create_project_dir dotenvfin + "$SECRETS_BIN" push dotenvfin >/dev/null 2>&1 + # no gradle-properties externals → v1count==0 path; no confirmation needed + run "$SECRETS_BIN" migrate --finalize + [ "$status" -eq 0 ] + [[ "$output" == *"no v1 blobs"* ]] || false + [ "$(cat "$SECRETS_DIR/.secrets-format")" = "2" ] +} + +@test "migrate with a positional argument dies" { + init_with_remote + create_project_dir mgpos + run "$SECRETS_BIN" migrate someproject + [ "$status" -eq 1 ] + [[ "$output" == *"no project argument"* ]] || false +} + +@test "finalize on an already-v2 store is a no-op" { + init_with_remote + create_project_dir finv2 + run "$SECRETS_BIN" migrate --finalize --yes + [ "$status" -eq 0 ] + [[ "$output" == *"already format v2"* ]] || false +} + +@test "_store_format reads a garbage marker as v1 (strict parse)" { + make_v1_store + create_project_dir garbagemarker + # a non-"2" marker (e.g. a truncated/garbled value) must read as v1, not v2 + printf 'v2-ish-garbage\n' > "$SECRETS_DIR/.secrets-format" + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"format: v1"* ]] || false +} From c10e89fb51e7d762c99ef728f40c809c04cc4457 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:47:17 -0700 Subject: [PATCH 21/70] fix: pre-landing review for store-format-v2 (verify-window orphan, copy-forward push, --force, comment) Pre-landing review (1 critical, 4 informational): - CRITICAL: per-project `secrets verify` during the migration window flagged the freshly-written v2 twin as a spurious orphan and exited 1 (store still reads v1, so verify's expected set only held the .gradle-properties.age name). Breaks the documented migrate->verify->finalize workflow and CI. Fix: verify's orphan set now accounts for BOTH suffix forms of a properties external, so the twin is never a false orphan mid-migration. Regression test added. - Copy-forward now pushes the twins (mirrors push/rekey) so a --finalize on another machine sees them; previously twins were local-only until finalize, a multi-machine footgun. - Dropped the undocumented `--force` alias (keep `--yes`). - Clarified the EGB-700 comment (which-format line, folded into EGB-703). Deferred to EGB-701: the two finalize find-walks over *.gradle-properties.age could collapse to one pass. Full suite 236/236. --- secrets | 16 ++++++++++++---- test/migrate.bats | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/secrets b/secrets index 2328fdc..97676b8 100755 --- a/secrets +++ b/secrets @@ -1908,8 +1908,8 @@ cmd_which() { resolve_store echo "store: $SECRETS_DIR" echo "source: $STORE_SOURCE" - # EGB-700: surface the store format so users can tell v1 from v2 during the - # migration window. A v1 store is a legacy store with no format marker. + # EGB-700 (folded into EGB-703): surface the store format so users can tell + # v1 from v2 during the migration window. v1 = legacy store, no format marker. echo "format: v$(_store_format)" # v2 manifest (.secrets.json): validate and summarize. Validation here @@ -2051,7 +2051,12 @@ _verify_project() { [ -n "$etype" ] || continue slug=$(_secrets_files_slug "$epath") erel="external/$slug.$(_external_blob_suffix "$etype").age" - expected="$expected$erel"$'\n' + # Account for BOTH the v1 and v2 suffix forms in the orphan set. During the + # migration window (after copy-forward, before --finalize) the v2 twin + # coexists with the v1 blob; neither should read as an orphan whichever + # format the store currently reports. (file's two forms are identical.) + expected="${expected}external/$slug.$etype.age"$'\n' + [ "$etype" = "gradle-properties" ] && expected="${expected}external/$slug.properties.age"$'\n' eblob="$pdir/$erel" if [ ! -f "$eblob" ]; then echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 @@ -2176,6 +2181,9 @@ _migrate_project() { ensure_store_protections git -C "$SECRETS_DIR" add -A git -C "$SECRETS_DIR" commit -m "migrate: copy-forward v2 twins for $project" >/dev/null 2>&1 || true + # Push the twins so a --finalize on another machine sees them (finalize + # refuses any v1 blob without a twin). Mirrors push/rekey's push behavior. + git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true info "Copy-forward for '$project': $moved new v2 twin(s), $already already present. v1 blobs kept (non-destructive). Run 'secrets migrate --finalize' once every project is migrated and every machine is upgraded." } @@ -2261,7 +2269,7 @@ cmd_migrate() { case "$1" in --dry-run) dry_run=true; shift ;; --finalize) finalize=true; shift ;; - --yes|--force) force=true; shift ;; + --yes) force=true; shift ;; -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --finalize] [--yes]" ;; *) die "migrate takes no project argument. Run it from inside a project (copy-forward) or use --finalize (store-wide)." ;; esac diff --git a/test/migrate.bats b/test/migrate.bats index 35102f6..84f7ec9 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -241,6 +241,21 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ ! -f "$SECRETS_DIR/.secrets-format" ] } +@test "verify is green during the migration window (v2 twin is not a spurious orphan)" { + # Regression: after copy-forward the store is still v1, so verify computed the + # external blob path as .gradle-properties.age and flagged the .properties.age + # twin as an orphan, failing verify mid-migration. The orphan set now accounts + # for both suffix forms. + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir windowverify + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push windowverify >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 + run "$SECRETS_BIN" verify windowverify + [ "$status" -eq 0 ] +} + @test "v1 client still reads during the migration window (after copy-forward, before finalize)" { make_v1_store m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' From cefa06280e2df4a9205a29dbf0b948907bacc1a4 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:56:27 -0700 Subject: [PATCH 22/70] chore: bump version and changelog (v0.6.0.0) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc91609..7d5167f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.6.0.0] - 2026-06-07 + +### Added + +- **Self-describing store format (v2) + `secrets migrate` (EGB-703)** — the + store now records its format in a committed `.secrets-format` file, and + `secrets which` prints it (`format: v2`). A fresh `secrets init` creates a + v2 store; existing stores read as v1 until migrated. +- **`secrets migrate`** — copy-forward a project's encrypted blobs to the v2 + layout. It is non-destructive: the old blobs are kept until you finalize, so + a half-migrated store stays fully readable and recoverable. `secrets migrate + --dry-run` previews exactly what would change without writing anything. +- **`secrets migrate --finalize`** — the one destructive step, run once + store-wide. It refuses unless `secrets verify` passes and every blob has its + new-format twin, cuts a `pre-v2-migrate-*` recovery tag first, then drops the + old blobs. It asks for confirmation (or `--yes`) because a machine still on + an older `secrets` will stop seeing migrated external files until it updates. + +### Changed + +- The external `properties` blob is stored as `.properties.age` in a v2 + store (was `.gradle-properties.age`), matching the manifest `type`. + `push`, `pull`, and `verify` pick the right name automatically from the store + format, so v1 and v2 stores both keep working during a migration. + ## [0.5.0.0] - 2026-06-07 ### Added diff --git a/VERSION b/VERSION index eddcc3f..fdae70d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0.0 +0.6.0.0 From c4da47f320778ea51e1448adf558b9a31ea453b0 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 16:57:53 -0700 Subject: [PATCH 23/70] docs: correct test counts for migrate.bats (236 total) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README.md: test suite count 210 → 236, "both files" → "three files". CLAUDE.md: testing command lists migrate.bats; Project Structure tree adds migrate.bats (EGB-703 store-format-v2 migration, 26 tests). Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 3 ++- README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e953300..2263b20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core -bats test/ # runs secrets.bats + manifest.bats +bats test/ # runs secrets.bats + manifest.bats + migrate.bats ./test/run-security.sh # security regression subset + operator sign-off (see below) ``` @@ -77,6 +77,7 @@ hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) manifest.bats # EGB-677 .secrets.json manifest tests (77 tests) + migrate.bats # EGB-703 store-format-v2 migration tests (26 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index 354e23c..10cf7dc 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (210 tests across both files) +# Run the test suite (236 tests across three files) brew install bats-core bats test/ From 94ee6ec9a28410c269d8a08c0fcb7fa85851930e Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 06:02:11 -0700 Subject: [PATCH 24/70] feat: secrets which prints the manifest version (EGB-700) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes EGB-700 — the store-format line (format: vN) shipped in v0.6.0.0; this adds the manifest schema version to the `which` manifest header (version N, validated == MANIFEST_VERSION by _check_manifest_file). Now a single `secrets which` surfaces both the store format and the manifest version for the dual-format debugging window. 1 bats test. Suite 237/237. --- secrets | 5 ++++- test/manifest.bats | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/secrets b/secrets index 97676b8..6b18cfe 100755 --- a/secrets +++ b/secrets @@ -1918,7 +1918,10 @@ cmd_which() { local json_manifest="$PWD/$SECRETS_JSON_NAME" if [ -e "$json_manifest" ]; then _check_manifest_file "$json_manifest" - echo "manifest ($SECRETS_JSON_NAME at $json_manifest):" + # EGB-700: surface the manifest schema version (validated == MANIFEST_VERSION + # by _check_manifest_file above) alongside the store format printed earlier. + local mver; mver=$(jq -r '.version // "?"' "$json_manifest") + echo "manifest ($SECRETS_JSON_NAME at $json_manifest, version $mver):" local entry while IFS= read -r entry; do [ -n "$entry" ] || continue diff --git a/test/manifest.bats b/test/manifest.bats index 37564c7..eb5c42a 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -97,6 +97,14 @@ load test_helper [[ "$output" == *".env"* ]] || false } +@test "which prints the manifest version (EGB-700)" { + create_project_dir manifestver + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"version 2"* ]] || false +} + @test "malformed .secrets.json dies with a directed error naming the file" { create_project_dir addproj echo '{ not json' > .secrets.json From 6a84846e6451e61653b503305529ae080aa7be07 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 06:19:57 -0700 Subject: [PATCH 25/70] chore: bump version and changelog (v0.6.0.1) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 9 +++++++++ VERSION | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5167f..026df25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.6.0.1] - 2026-06-08 + +### Added + +- **`secrets which` now prints the manifest version (EGB-700)** — the manifest + header line shows `version N` alongside the store format, so a single + `secrets which` tells you both the on-disk store format and the `.secrets.json` + schema version at a glance. + ## [0.6.0.0] - 2026-06-07 ### Added diff --git a/VERSION b/VERSION index fdae70d..758efdb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0.0 +0.6.0.1 From 2549832f0e06e45c04429054c095aaaf325adee4 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 06:21:26 -0700 Subject: [PATCH 26/70] docs: sync test counts and which manifest-version note (EGB-700) - README: test suite total 236 -> 237 - CLAUDE.md: manifest.bats 77 -> 78 tests - CLAUDE.md: note `secrets which` now prints the .secrets.json schema version in its manifest header line alongside `format: vN` Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2263b20..ece6714 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. -- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints `format: vN`. **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, writes `.properties.age` twins beside the v1 blobs; needs the project manifest to know which externals are `properties`; idempotent) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, writes `.properties.age` twins beside the v1 blobs; needs the project manifest to know which externals are `properties`; idempotent) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` @@ -76,7 +76,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (77 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (78 tests) migrate.bats # EGB-703 store-format-v2 migration tests (26 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation diff --git a/README.md b/README.md index 10cf7dc..50fb49f 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (236 tests across three files) +# Run the test suite (237 tests across three files) brew install bats-core bats test/ From 679ddbc1b7945470e2dd3626c64a322598362700 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 06:46:50 -0700 Subject: [PATCH 27/70] docs: EGB-710 implementation plan (migrate guided flow) --- .../2026-06-08-egb-710-migrate-guided-flow.md | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-08-egb-710-migrate-guided-flow.md diff --git a/docs/superpowers/plans/2026-06-08-egb-710-migrate-guided-flow.md b/docs/superpowers/plans/2026-06-08-egb-710-migrate-guided-flow.md new file mode 100644 index 0000000..b552428 --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-egb-710-migrate-guided-flow.md @@ -0,0 +1,447 @@ +# EGB-710: `secrets migrate` guided flow (manifest-free copy-forward + `--status` survey) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the dead-end "No .secrets.json — cd into a project that has a manifest" error in `secrets migrate` with a manifest-free copy-forward that just works, plus a `secrets migrate --status` survey that tells the operator exactly which projects still need migrating. + +**Architecture:** The per-project copy-forward step (`_migrate_project`) currently reads `$PWD/.secrets.json` to find `properties` externals, then looks for their v1 blobs. This makes it die on a legacy `.secrets-files`-only project (the real EGB-710 repro) and — worse — silently skips any store blob the manifest doesn't declare, leaving it un-twinned so `--finalize` later refuses it. The fix: enumerate the **store** (`$SECRETS_DIR//external/*.gradle-properties.age`) directly and copy each to its `.properties.age` twin via suffix-swap — the *exact* logic `_migrate_finalize` already uses. No manifest needed, dotenv/file-only projects become a clean no-op, and migrate twins precisely the blobs finalize will demand twins for. A new `--status` mode walks every project dir and reports per-project readiness + whether the store is finalize-ready. + +**Tech Stack:** Single bash 3.2 script (`secrets`), `age`, `git`, `jq` (unaffected here). Tests: `bats-core` (`test/migrate.bats`). Every standalone `[[ ]]` assertion ends with `|| false` (bash 3.2 ERR-trap gotcha). + +--- + +## Background facts (verified against HEAD 633d19e) + +- `_migrate_project()` lives at `secrets:2135`; it `die`s at `secrets:2142-2148` on missing `$PWD/.secrets.json`, then iterates `_json_external_entries "$manifest"` filtered to `gradle-properties` (`secrets:2154-2174`). +- `_migrate_finalize()` (`secrets:2196`) already enumerates blobs manifest-free: `find "$SECRETS_DIR" -type f -name '*.gradle-properties.age'` and derives the twin as `new="${f%.gradle-properties.age}.properties.age"` (`secrets:2212-2217`). The per-project step will mirror this, scoped to one project dir. +- `cmd_migrate()` flag parser: `secrets:2267-2285`. +- `derive_project_name ""` (`secrets:101`) needs no manifest — it uses the git remote basename or `basename "$PWD"`. +- `info()`/`die()`: `secrets:36-37`. `ensure_store_protections` is called post-write in the existing copy-forward. +- Output-contract strings existing tests depend on (must be preserved): `"would migrate"` (migrate.bats:80), `"0 blob(s) would be copy-forwarded"` (migrate.bats:94), `"1 already present"` (migrate.bats:121, idempotent re-run prints `$already already present`), `"already format v2"` (migrate.bats:139). +- The one test that codifies the OLD dead-end — `"migrate with no manifest in cwd dies with a directed message"` (migrate.bats:126) — is the behavior we are intentionally changing; it gets rewritten in Task 1. +- No test asserts the non-dry-run "nothing to migrate" wording (grep confirmed), so that message is free to change. We keep the substring `no v1 properties blobs` regardless for safety. +- VERSION is `0.6.0.1`; bump to `0.6.1.0` (new subcommand flag + behavior change). `MANIFEST_VERSION` stays `2` (no schema change). + +## File structure + +- Modify: `secrets` — rewrite `_migrate_project` (`secrets:2135-2191`), add `_migrate_status`, extend `cmd_migrate` flag parsing + dispatch (`secrets:2267-2285`), update `cmd_help` migrate lines (`secrets:2308-2309`). +- Modify: `test/migrate.bats` — rewrite the dead-end test; add manifest-free, finalize-consistency, and `--status` tests. +- Modify: `CLAUDE.md` — update the migrate paragraph (Architecture → Store format) to note manifest-free copy-forward + `--status`. +- Modify: `README.md` — migrate usage/help. +- Modify: `VERSION` → `0.6.1.0`; `CHANGELOG.md` — new entry. + +--- + +## Task 1: Manifest-free per-project copy-forward + +**Files:** +- Modify: `secrets` — `_migrate_project()` (`secrets:2135-2191`) +- Test: `test/migrate.bats` + +- [ ] **Step 1: Write the failing test — migrate works with no `.secrets.json` (the EGB-710 repro)** + +Add to `test/migrate.bats` (after the existing copy-forward tests, ~line 124): + +```bash +@test "migrate copy-forwards a v1 properties blob with no .secrets.json (manifest-free)" { + # The EGB-710 repro: a legacy project has a v1 properties blob in the store + # but no .secrets.json (it predates the manifest). migrate must NOT dead-end. + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir nomanifestblob + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push nomanifestblob >/dev/null 2>&1 + rm -f .secrets.json # simulate a pre-manifest project + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/nomanifestblob/external/*.properties.age" + [ "$status" -eq 0 ] # v2 twin written despite no manifest + run bash -c "ls $SECRETS_DIR/nomanifestblob/external/*.gradle-properties.age" + [ "$status" -eq 0 ] # v1 kept (non-destructive) +} +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `bats test/migrate.bats -f "manifest-free"` +Expected: FAIL — current code `die`s with "No .secrets.json in …" (status 1), so the first `[ "$status" -eq 0 ]` fails. + +- [ ] **Step 3: Rewrite `_migrate_project` to enumerate the store, not the manifest** + +Replace the body of `_migrate_project()` (`secrets:2135-2191`, from `_migrate_project() {` through its closing `}`) with: + +```bash +_migrate_project() { + local dry_run="$1" + if [ "$(_store_format)" = "2" ]; then + info "Store is already format v2 — nothing to migrate." + return 0 + fi + local project; project=$(derive_project_name "") + local pdir="$SECRETS_DIR/$project" + + # Source of truth for the copy-forward is the STORE, not a project manifest. + # Every v1 properties blob is a `*.gradle-properties.age` file whose v2 twin + # is the same name with the `.properties.age` suffix (the only on-disk change + # v2 makes). Enumerating the store — exactly as `_migrate_finalize` does — + # means migrate twins precisely the blobs finalize will demand twins for, with + # no manifest dependency. This is why a legacy `.secrets-files`-only project + # (no `.secrets.json` yet) migrates cleanly instead of dead-ending, and why a + # store blob the manifest no longer declares still gets a twin. + local moved=0 already=0 would=0 v1count=0 f new + while IFS= read -r f; do + [ -f "$f" ] || continue + v1count=$((v1count + 1)) + new="${f%.gradle-properties.age}.properties.age" + if [ -f "$new" ]; then + already=$((already + 1)) + continue + fi + if [ "$dry_run" = true ]; then + echo "would migrate: ${f#"$SECRETS_DIR"/} -> $(basename "$new")" + would=$((would + 1)) + else + cp "$f" "$new" + moved=$((moved + 1)) + fi + done < <(find "$pdir/external" -type f -name '*.gradle-properties.age' 2>/dev/null) + + if [ "$dry_run" = true ]; then + echo "migrate --dry-run: $would blob(s) would be copy-forwarded for '$project' (writes nothing); $already already present. v1 blobs are kept until 'secrets migrate --finalize'." + return 0 + fi + if [ "$moved" -eq 0 ] && [ "$already" -eq 0 ]; then + info "Nothing to migrate for '$project' — no v1 properties blobs in the store (already v2-shaped). If you expected one, run 'secrets push' first, then re-run 'secrets migrate'." + return 0 + fi + ensure_store_protections + git -C "$SECRETS_DIR" add -A + git -C "$SECRETS_DIR" commit -m "migrate: copy-forward v2 twins for $project" >/dev/null 2>&1 || true + # Push the twins so a --finalize on another machine sees them (finalize + # refuses any v1 blob without a twin). Mirrors push/rekey's push behavior. + git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true + info "Copy-forward for '$project': $moved new v2 twin(s), $already already present. v1 blobs kept (non-destructive). Run 'secrets migrate --finalize' once every project is migrated and every machine is upgraded." +} +``` + +Notes for the implementer: +- This removes the only in-script caller of `_check_manifest_file`/`_json_external_entries` *inside migrate*; both remain defined and used by push/pull/verify, so do **not** delete them. +- `${f#"$SECRETS_DIR"/}` keeps the dry-run line's store-relative form (matches the old `$project/external/...` style closely enough; the test only checks the substring `would migrate`). +- The `moved==0 && already==0` branch keeps the substring `no v1 properties blobs`. The idempotent re-run path (`moved==0, already>0`) falls through to the final `info` printing `$already already present`, preserving the `1 already present` contract. + +- [ ] **Step 4: Run the new test + the full migrate suite** + +Run: `bats test/migrate.bats` +Expected: the new "manifest-free" test PASSES; existing dry-run/copy-forward/idempotent/finalize tests still PASS; the "migrate with no manifest in cwd dies" test (migrate.bats:126) now FAILS (we fix it in Step 5). + +- [ ] **Step 5: Update the test that codified the old dead-end** + +Replace the test at `test/migrate.bats:126-132` (`"migrate with no manifest in cwd dies with a directed message"`) with: + +```bash +@test "migrate in a project with no manifest and no store blobs is a clean no-op" { + make_v1_store + local dir="$WORK_DIR/nomanifest"; mkdir -p "$dir"; cd "$dir" + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + [[ "$output" == *"no v1 properties blobs"* ]] || false +} +``` + +- [ ] **Step 6: Run the full suite to confirm green** + +Run: `bats test/migrate.bats` +Expected: all PASS. + +- [ ] **Step 7: Commit** + +```bash +git add secrets test/migrate.bats +git commit -m "fix: migrate copy-forward is manifest-free, no dead-end on legacy projects (EGB-710)" +``` + +--- + +## Task 2: Finalize-consistency regression test (store blob not in manifest) + +**Files:** +- Test: `test/migrate.bats` + +This proves the latent-bug fix: the old manifest-driven migrate skipped store blobs the manifest didn't declare, leaving them un-twinned so `--finalize` refused them. The rewrite twins them. + +- [ ] **Step 1: Write the test** + +Add to `test/migrate.bats`: + +```bash +@test "migrate twins a store blob even when the manifest no longer declares it" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir staleblob + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push staleblob >/dev/null 2>&1 + # The blob is now in the store. Drop the external from the project's manifest + # entirely (and remove the legacy file) so NO manifest declares it. + printf '{"version":2,"dotenv":[".env",".env.staging"]}\n' > .secrets.json + rm -f .secrets-files + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/staleblob/external/*.properties.age" + [ "$status" -eq 0 ] # twinned despite not being declared anywhere +} +``` + +- [ ] **Step 2: Run it** + +Run: `bats test/migrate.bats -f "no longer declares"` +Expected: PASS (the Task 1 rewrite already makes this green — this test guards against regressing back to manifest-driven enumeration). + +- [ ] **Step 3: Commit** + +```bash +git add test/migrate.bats +git commit -m "test: migrate twins undeclared store blobs (finalize-consistency, EGB-710)" +``` + +--- + +## Task 3: `secrets migrate --status` survey + +**Files:** +- Modify: `secrets` — add `_migrate_status()`; extend `cmd_migrate` (`secrets:2267-2285`) +- Test: `test/migrate.bats` + +- [ ] **Step 1: Write the failing tests** + +Add to `test/migrate.bats`: + +```bash +@test "migrate --status flags a project that needs migrating" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir needsmig + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push needsmig >/dev/null 2>&1 # v1 blob, no twin yet + run "$SECRETS_BIN" migrate --status + [ "$status" -ne 0 ] # not finalize-ready + [[ "$output" == *"needsmig"* ]] || false + [[ "$output" == *"NEEDS MIGRATE"* ]] || false + [[ "$output" == *"Not finalize-ready"* ]] || false +} + +@test "migrate --status reports finalize-ready once every blob is twinned" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir readymig + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push readymig >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 # create the twin + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] + [[ "$output" == *"Finalize-ready"* ]] || false +} + +@test "migrate --status on an already-v2 store says nothing to do" { + init_with_remote + create_project_dir v2status + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] + [[ "$output" == *"v2"* ]] || false +} +``` + +- [ ] **Step 2: Run them to confirm they fail** + +Run: `bats test/migrate.bats -f "status"` +Expected: FAIL — `--status` is an unknown flag today (`die "Unknown migrate flag: --status…"`, status 1), so the assertions fail. + +- [ ] **Step 3: Add `_migrate_status` (place it just before `_migrate_finalize`, ~`secrets:2195`)** + +```bash +# Read-only survey: walk every project dir in the store and report each one's +# v2-readiness from the blobs on disk (no manifest, no decryption). Exits +# non-zero when any v1 properties blob lacks a v2 twin (i.e. the store is not +# yet finalize-ready) so it can gate scripting, mirroring `verify`'s posture. +_migrate_status() { + if [ "$(_store_format)" = "2" ]; then + info "Store format: v2 (finalized) — nothing to migrate." + return 0 + fi + echo "Store format: v1 (not finalized). Per-project migration status:" + local any_untwinned=0 dir project v1 untwinned f new + for dir in "$SECRETS_DIR"/*/; do + [ -d "$dir" ] || continue + project=$(basename "$dir") + case "$project" in .*) continue ;; esac + v1=0; untwinned=0 + while IFS= read -r f; do + [ -f "$f" ] || continue + v1=$((v1 + 1)) + new="${f%.gradle-properties.age}.properties.age" + [ -f "$new" ] || untwinned=$((untwinned + 1)) + done < <(find "$dir" -type f -name '*.gradle-properties.age' 2>/dev/null) + if [ "$v1" -eq 0 ]; then + echo " $project: v2-ready (no v1 properties blobs)" + elif [ "$untwinned" -eq 0 ]; then + echo " $project: migrated ($v1 v1 blob(s), all twinned)" + else + echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'" + any_untwinned=1 + fi + done + echo + if [ "$any_untwinned" -eq 1 ]; then + echo "Not finalize-ready: migrate the projects marked NEEDS MIGRATE, then run 'secrets migrate --finalize'." + return 1 + fi + echo "Finalize-ready: every v1 properties blob has a v2 twin. Run 'secrets migrate --finalize' once every machine is upgraded." + return 0 +} +``` + +- [ ] **Step 4: Wire `--status` into `cmd_migrate`** + +In `cmd_migrate()` (`secrets:2267`), add the `status` local and parse + dispatch. Replace: + +```bash + local dry_run=false finalize=false force=false + while [ $# -gt 0 ]; do + case "$1" in + --dry-run) dry_run=true; shift ;; + --finalize) finalize=true; shift ;; + --yes) force=true; shift ;; + -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --finalize] [--yes]" ;; + *) die "migrate takes no project argument. Run it from inside a project (copy-forward) or use --finalize (store-wide)." ;; + esac + done + if [ "$finalize" = true ]; then + _migrate_finalize "$force" + else + _migrate_project "$dry_run" + fi +``` + +with: + +```bash + local dry_run=false finalize=false force=false status=false + while [ $# -gt 0 ]; do + case "$1" in + --dry-run) dry_run=true; shift ;; + --finalize) finalize=true; shift ;; + --status) status=true; shift ;; + --yes) force=true; shift ;; + -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --status | --finalize] [--yes]" ;; + *) die "migrate takes no project argument. Run it from inside a project (copy-forward), or use --status / --finalize (store-wide)." ;; + esac + done + if [ "$status" = true ]; then + _migrate_status + elif [ "$finalize" = true ]; then + _migrate_finalize "$force" + else + _migrate_project "$dry_run" + fi +``` + +- [ ] **Step 5: Run the status tests + full suite** + +Run: `bats test/migrate.bats` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +git add secrets test/migrate.bats +git commit -m "feat: secrets migrate --status surveys per-project v2 readiness (EGB-710)" +``` + +--- + +## Task 4: Docs, help text, version, changelog + +**Files:** +- Modify: `secrets` — `cmd_help` (`secrets:2308-2309`) +- Modify: `CLAUDE.md`, `README.md`, `VERSION`, `CHANGELOG.md` + +- [ ] **Step 1: Update `cmd_help` migrate lines** + +In `cmd_help()` replace the two migrate lines (`secrets:2308-2309`): + +``` + secrets migrate [--dry-run] Copy-forward this project's blobs to store format v2 + secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) +``` + +with: + +``` + secrets migrate [--dry-run] Copy-forward this project's v1 blobs to store format v2 + secrets migrate --status Survey every project's v2 readiness (finalize gate) + secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) +``` + +- [ ] **Step 2: Update `CLAUDE.md` migrate paragraph** + +In the "Store format (EGB-677 stage 2 / EGB-703)" bullet, update the migration description: copy-forward is now **manifest-free** — `secrets migrate` enumerates the project's `*.gradle-properties.age` store blobs directly (same source of truth as `--finalize`), so a legacy `.secrets-files`-only project migrates without a `.secrets.json` and no store blob is left un-twinned. Add `secrets migrate --status` to the workflow line as the read-only survey that reports per-project readiness and gates `--finalize`. Reference EGB-710. + +- [ ] **Step 3: Update `README.md`** + +Find the migrate section/help block and add the `--status` line and the manifest-free note, mirroring the help text. + +- [ ] **Step 4: Bump `VERSION`** + +Set `VERSION` to `0.6.1.0`. + +- [ ] **Step 5: Add `CHANGELOG.md` entry** + +Insert above `## [0.6.0.1]`: + +```markdown +## [0.6.1.0] - 2026-06-08 + +### Changed + +- **`secrets migrate` copy-forward is now manifest-free (EGB-710)** — the + per-project step enumerates the store's `*.gradle-properties.age` blobs + directly (the same source of truth `--finalize` uses) instead of reading + `.secrets.json`. A legacy `.secrets-files`-only project now migrates cleanly + instead of dead-ending with "No .secrets.json", and a store blob the manifest + no longer declares still gets a v2 twin (so `--finalize` won't refuse it). + Running migrate in a project with no v1 properties blobs is a clean no-op. + +### Added + +- **`secrets migrate --status`** — a read-only survey that walks every project + in the store and reports its v2 readiness (v2-ready / migrated / NEEDS + MIGRATE), then whether the store as a whole is finalize-ready. Exits non-zero + while any v1 blob is un-twinned, so it can gate the path to `--finalize`. +``` + +- [ ] **Step 6: Run the full bats suite (all files)** + +Run: `bats test/` +Expected: all PASS (secrets.bats + manifest.bats + migrate.bats). Confirm the count went up by the tests added here. + +- [ ] **Step 7: Commit** + +```bash +git add secrets CLAUDE.md README.md VERSION CHANGELOG.md +git commit -m "docs: migrate --status + manifest-free copy-forward; bump 0.6.1.0 (EGB-710)" +``` + +--- + +## Self-review against the EGB-710 spec + +- **AC: dotenv-only project exits 0 "already v2-shaped"** → Task 1 (`moved==0 && already==0` branch, substring `no v1 properties blobs`). Test: "no manifest and no store blobs is a clean no-op" (Task 1 Step 5). +- **AC: `.secrets-files`-only project with v1 blobs migrates instead of the generic error** → the manifest-free rewrite makes it *just migrate* (better than guiding to push first). Test: "manifest-free" (Task 1 Step 1). Note the rewrite supersedes the ticket's "offer to run absorb then migrate" branch — migrate no longer needs the manifest, so there's nothing to prompt for; the only destructive step (`--finalize`) keeps its existing `/dev/tty` confirmation. +- **AC: interactive prompts read `/dev/tty`, degrade non-interactively, bash 3.2** → no new prompt is introduced (copy-forward is non-destructive); `--finalize`'s existing `/dev/tty` confirm is untouched. `--status` is pure read-only output. All new code is bash 3.2 (no associative arrays, `[[ ]]` only in tests with `|| false`). +- **AC: no behavior change to copy-forward/finalize safety gates** → finalize untouched; copy-forward stays non-destructive (v1 kept, commit + push twins). Verified by the unchanged finalize tests (migrate.bats:168-214). +- **AC: bats coverage per branch** → Tasks 1-3 add: manifest-free migrate, no-op no-blobs, undeclared-blob twinning, `--status` needs-migrate / finalize-ready / already-v2. +- **Stretch: `--status` survey** → Task 3, delivered. +- **Sequencing: independent of the EGB-709 gate** → confirmed; this only touches migrate ergonomics and helps operators *reach* all-v2. EGB-703 safety posture (recovery tag, verify-green gate, twin-before-drop) is preserved. + +## Operator-local follow-up (not part of this plan) + +This repo's `.ship-policy.json` opts out of AI security/red-team passes; before any PR, ask the operator to run `./test/run-security.sh` and complete the SIGNOFF. After landing, the operator can re-run their real-store migration for `onefinalmessage` et al. (`secrets migrate` per project → `secrets migrate --finalize`), which is the path to clearing the EGB-709 v2 gate. From c121982dcd91e385c88082efdd1ded954654011a Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 06:58:22 -0700 Subject: [PATCH 28/70] fix: migrate copy-forward is manifest-free, no dead-end on legacy projects (EGB-710) --- secrets | 41 +++++++++++++++++------------------------ test/migrate.bats | 23 ++++++++++++++++++++--- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/secrets b/secrets index 6b18cfe..e4b86d3 100755 --- a/secrets +++ b/secrets @@ -2138,47 +2138,40 @@ _migrate_project() { info "Store is already format v2 — nothing to migrate." return 0 fi - local manifest="$PWD/$SECRETS_JSON_NAME" - if [ ! -e "$manifest" ]; then - die "No $SECRETS_JSON_NAME in $PWD. - 'secrets migrate' copy-forwards a project's v1 blobs to their v2 names and - reads the project manifest to do so. cd into a project that has a manifest, - then run 'secrets migrate'. (Store-wide 'secrets migrate --finalize' comes - after every project is migrated.)" - fi - _check_manifest_file "$manifest" local project; project=$(derive_project_name "") local pdir="$SECRETS_DIR/$project" - local moved=0 already=0 would=0 etype epath slug old new - while IFS=$'\t' read -r etype epath _; do - [ -n "$etype" ] || continue - # Only `properties` blobs change name in v2; dotenv and `file` are already - # in their v2 shape and never move. - [ "$etype" = "gradle-properties" ] || continue - slug=$(_secrets_files_slug "$epath") - old="$pdir/external/$slug.gradle-properties.age" - new="$pdir/external/$slug.properties.age" - [ -f "$old" ] || continue # nothing pushed yet (or already dropped) - if [ -f "$new" ]; then # idempotent: twin already exists + # Source of truth for the copy-forward is the STORE, not a project manifest. + # Every v1 properties blob is a `*.gradle-properties.age` file whose v2 twin + # is the same name with the `.properties.age` suffix (the only on-disk change + # v2 makes). Enumerating the store — exactly as `_migrate_finalize` does — + # means migrate twins precisely the blobs finalize will demand twins for, with + # no manifest dependency. This is why a legacy `.secrets-files`-only project + # (no `.secrets.json` yet) migrates cleanly instead of dead-ending, and why a + # store blob the manifest no longer declares still gets a twin. + local moved=0 already=0 would=0 f new + while IFS= read -r f; do + [ -f "$f" ] || continue + new="${f%.gradle-properties.age}.properties.age" + if [ -f "$new" ]; then already=$((already + 1)) continue fi if [ "$dry_run" = true ]; then - echo "would migrate: $project/external/$slug.gradle-properties.age -> $slug.properties.age" + echo "would migrate: ${f#"$SECRETS_DIR"/} -> $(basename "$new")" would=$((would + 1)) else - cp "$old" "$new" + cp "$f" "$new" moved=$((moved + 1)) fi - done < <(_json_external_entries "$manifest") + done < <(find "$pdir/external" -type f -name '*.gradle-properties.age' 2>/dev/null) if [ "$dry_run" = true ]; then echo "migrate --dry-run: $would blob(s) would be copy-forwarded for '$project' (writes nothing); $already already present. v1 blobs are kept until 'secrets migrate --finalize'." return 0 fi if [ "$moved" -eq 0 ] && [ "$already" -eq 0 ]; then - info "Nothing to migrate for '$project' (no v1 properties blobs)." + info "Nothing to migrate for '$project' — no v1 properties blobs in the store (already v2-shaped). If you expected one, run 'secrets push' first, then re-run 'secrets migrate'." return 0 fi ensure_store_protections diff --git a/test/migrate.bats b/test/migrate.bats index 84f7ec9..cd7dd30 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -123,12 +123,29 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$output" = "1" ] } -@test "migrate with no manifest in cwd dies with a directed message" { +@test "migrate copy-forwards a v1 properties blob with no .secrets.json (manifest-free)" { + # The EGB-710 repro: a legacy project has a v1 properties blob in the store + # but no .secrets.json (it predates the manifest). migrate must NOT dead-end. + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir nomanifestblob + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push nomanifestblob >/dev/null 2>&1 + rm -f .secrets.json # simulate a pre-manifest project + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/nomanifestblob/external/*.properties.age" + [ "$status" -eq 0 ] # v2 twin written despite no manifest + run bash -c "ls $SECRETS_DIR/nomanifestblob/external/*.gradle-properties.age" + [ "$status" -eq 0 ] # v1 kept (non-destructive) +} + +@test "migrate in a project with no manifest and no store blobs is a clean no-op" { make_v1_store local dir="$WORK_DIR/nomanifest"; mkdir -p "$dir"; cd "$dir" run "$SECRETS_BIN" migrate - [ "$status" -eq 1 ] - [[ "$output" == *".secrets.json"* ]] || false + [ "$status" -eq 0 ] + [[ "$output" == *"no v1 properties blobs"* ]] || false } @test "migrate on an already-v2 store is a no-op" { From e75627deff575b016e442fe07b4486a358ee6de8 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 07:18:22 -0700 Subject: [PATCH 29/70] test: migrate twins undeclared store blobs (finalize-consistency, EGB-710) --- test/migrate.bats | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/migrate.bats b/test/migrate.bats index cd7dd30..f3d2daa 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -140,6 +140,24 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -eq 0 ] # v1 kept (non-destructive) } +@test "migrate twins a store blob even when the manifest no longer declares it" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir staleblob + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push staleblob >/dev/null 2>&1 + # The blob is now in the store. Drop the external from the project's manifest + # entirely (and remove the legacy file) so NO manifest declares it. + printf '{"version":2,"dotenv":[".env",".env.staging"]}\n' > .secrets.json + rm -f .secrets-files + run bash -c "ls $SECRETS_DIR/staleblob/external/*.gradle-properties.age" + [ "$status" -eq 0 ] # precondition: the v1 blob exists in the store + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/staleblob/external/*.properties.age" + [ "$status" -eq 0 ] # twinned despite not being declared anywhere +} + @test "migrate in a project with no manifest and no store blobs is a clean no-op" { make_v1_store local dir="$WORK_DIR/nomanifest"; mkdir -p "$dir"; cd "$dir" From 469b08203b12dc7203f11ed3fdcdeeca58baad46 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 07:34:55 -0700 Subject: [PATCH 30/70] feat: secrets migrate --status surveys per-project v2 readiness (EGB-710) --- secrets | 51 +++++++++++++++++++++++++++++++++++++++++++---- test/migrate.bats | 34 +++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/secrets b/secrets index e4b86d3..6f4158a 100755 --- a/secrets +++ b/secrets @@ -2186,6 +2186,46 @@ _migrate_project() { # Store-wide finalize: the only destructive step. Refuses unless verify --all # is green and every v1 properties blob has a v2 twin. Cuts a recovery tag, # stamps the marker, then drops v1 blobs. +# Read-only survey: walk every project dir in the store and report each one's +# v2-readiness from the blobs on disk (no manifest, no decryption). Exits +# non-zero when any v1 properties blob lacks a v2 twin (i.e. the store is not +# yet finalize-ready) so it can gate scripting, mirroring `verify`'s posture. +_migrate_status() { + if [ "$(_store_format)" = "2" ]; then + info "Store format: v2 (finalized) — nothing to migrate." + return 0 + fi + echo "Store format: v1 (not finalized). Per-project migration status:" + local any_untwinned=0 dir project v1 untwinned f new + for dir in "$SECRETS_DIR"/*/; do + [ -d "$dir" ] || continue + project=$(basename "$dir") + case "$project" in .*) continue ;; esac + v1=0; untwinned=0 + while IFS= read -r f; do + [ -f "$f" ] || continue + v1=$((v1 + 1)) + new="${f%.gradle-properties.age}.properties.age" + [ -f "$new" ] || untwinned=$((untwinned + 1)) + done < <(find "$dir" -type f -name '*.gradle-properties.age' 2>/dev/null) + if [ "$v1" -eq 0 ]; then + echo " $project: v2-ready (no v1 properties blobs)" + elif [ "$untwinned" -eq 0 ]; then + echo " $project: migrated ($v1 v1 blob(s), all twinned)" + else + echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'" + any_untwinned=1 + fi + done + echo + if [ "$any_untwinned" -eq 1 ]; then + echo "Not finalize-ready: migrate the projects marked NEEDS MIGRATE, then run 'secrets migrate --finalize'." + return 1 + fi + echo "Finalize-ready: every v1 properties blob has a v2 twin. Run 'secrets migrate --finalize' once every machine is upgraded." + return 0 +} + _migrate_finalize() { local force="$1" check_key @@ -2260,17 +2300,20 @@ $untwinned cd into each project and run 'secrets migrate', then re-run 'secrets cmd_migrate() { resolve_store check_initialized - local dry_run=false finalize=false force=false + local dry_run=false finalize=false force=false status=false while [ $# -gt 0 ]; do case "$1" in --dry-run) dry_run=true; shift ;; --finalize) finalize=true; shift ;; + --status) status=true; shift ;; --yes) force=true; shift ;; - -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --finalize] [--yes]" ;; - *) die "migrate takes no project argument. Run it from inside a project (copy-forward) or use --finalize (store-wide)." ;; + -*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --status | --finalize] [--yes]" ;; + *) die "migrate takes no project argument. Run it from inside a project (copy-forward), or use --status / --finalize (store-wide)." ;; esac done - if [ "$finalize" = true ]; then + if [ "$status" = true ]; then + _migrate_status + elif [ "$finalize" = true ]; then _migrate_finalize "$force" else _migrate_project "$dry_run" diff --git a/test/migrate.bats b/test/migrate.bats index f3d2daa..9441bca 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -346,6 +346,40 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [[ "$output" == *"already format v2"* ]] || false } +@test "migrate --status flags a project that needs migrating" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir needsmig + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push needsmig >/dev/null 2>&1 # v1 blob, no twin yet + run "$SECRETS_BIN" migrate --status + [ "$status" -ne 0 ] # not finalize-ready + [[ "$output" == *"needsmig"* ]] || false + [[ "$output" == *"NEEDS MIGRATE"* ]] || false + [[ "$output" == *"Not finalize-ready"* ]] || false +} + +@test "migrate --status reports finalize-ready once every blob is twinned" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir readymig + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push readymig >/dev/null 2>&1 + "$SECRETS_BIN" migrate >/dev/null 2>&1 # create the twin + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] + [[ "$output" == *"Finalize-ready"* ]] || false +} + +@test "migrate --status on an already-v2 store says nothing to do" { + init_with_remote + create_project_dir v2status + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] + [[ "$output" == *"v2"* ]] || false + [[ "$output" == *"nothing to migrate"* ]] || false +} + @test "_store_format reads a garbage marker as v1 (strict parse)" { make_v1_store create_project_dir garbagemarker From 5729d84e75a416d97798ad087212c9bd5b9f701c Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 08:01:44 -0700 Subject: [PATCH 31/70] docs: migrate --status + manifest-free copy-forward; bump 0.6.1.0 (EGB-710) --- CHANGELOG.md | 19 +++++++++++++++++++ CLAUDE.md | 2 +- README.md | 3 ++- VERSION | 2 +- secrets | 3 ++- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 026df25..774c55a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.6.1.0] - 2026-06-08 + +### Changed + +- **`secrets migrate` copy-forward is now manifest-free (EGB-710)** — the + per-project step enumerates the store's `*.gradle-properties.age` blobs + directly (the same source of truth `--finalize` uses) instead of reading + `.secrets.json`. A legacy `.secrets-files`-only project now migrates cleanly + instead of dead-ending with "No .secrets.json", and a store blob the manifest + no longer declares still gets a v2 twin (so `--finalize` won't refuse it). + Running migrate in a project with no v1 properties blobs is a clean no-op. + +### Added + +- **`secrets migrate --status`** — a read-only survey that walks every project + in the store and reports its v2 readiness (v2-ready / migrated / NEEDS + MIGRATE), then whether the store as a whole is finalize-ready. Exits non-zero + while any v1 blob is un-twinned, so it can gate the path to `--finalize`. + ## [0.6.0.1] - 2026-06-08 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index ece6714..176fd12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. -- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, writes `.properties.age` twins beside the v1 blobs; needs the project manifest to know which externals are `properties`; idempotent) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` diff --git a/README.md b/README.md index 50fb49f..ee177b3 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,8 @@ secrets clear | `secrets rekey` | Generate a new encryption key and re-encrypt everything | | `secrets verify [project]` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob. `[project]` overrides the store directory name; the manifest is still read from the current directory | | `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | -| `secrets migrate [--dry-run]` | Copy-forward this project's encrypted blobs to store format v2 (non-destructive; `--dry-run` previews) | +| `secrets migrate [--dry-run]` | Copy-forward this project's encrypted blobs to store format v2 (non-destructive; manifest-free; `--dry-run` previews) | +| `secrets migrate --status` | Survey every project's v2 readiness; exits non-zero until the whole store is finalize-ready | | `secrets migrate --finalize` | Drop the old v1 blobs and mark the store v2 — runs once, store-wide, after `verify` is green and every machine is upgraded | ### Automatic project detection diff --git a/VERSION b/VERSION index 758efdb..44e7f9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0.1 +0.6.1.0 diff --git a/secrets b/secrets index 6f4158a..02697d4 100755 --- a/secrets +++ b/secrets @@ -2341,7 +2341,8 @@ Usage: secrets rekey Re-encrypt all secrets with a new key secrets verify [project] Check the manifest against the store + decrypt every blob secrets verify --all Decrypt-test every blob in every project (integrity gate) - secrets migrate [--dry-run] Copy-forward this project's blobs to store format v2 + secrets migrate [--dry-run] Copy-forward this project's v1 blobs to store format v2 + secrets migrate --status Survey every project's v2 readiness (finalize gate) secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) secrets which Show the active store, manifest, and external entries secrets where Alias for `which` From 3ac2a86a943e639fd04f19651a6b62b0057f8b7b Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 08:26:16 -0700 Subject: [PATCH 32/70] docs: fix stale manifest-dependency comments in migrate (EGB-710) --- secrets | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/secrets b/secrets index 02697d4..02a3415 100755 --- a/secrets +++ b/secrets @@ -2127,11 +2127,11 @@ cmd_verify() { # secrets migrate --dry-run # per project: report old→new, write nothing # secrets migrate # per project: write v2 twins beside v1 blobs # secrets migrate --finalize # store-wide: verify, drop v1, stamp v2 -# Per-project (needs the project manifest to know which externals are -# `properties`); finalize is store-wide. Mirrors verify's project/--all split. +# Per-project copy-forward is manifest-free (enumerates the store's blobs); +# finalize is store-wide. Mirrors verify's project/--all split. -# Copy-forward (or dry-run preview) for the current project. Reads -# $PWD/.secrets.json; only `properties` external blobs rename in v2. +# Copy-forward (or dry-run preview) for the current project. Manifest-free: +# enumerates the store's `*.gradle-properties.age` blobs; only those rename in v2. _migrate_project() { local dry_run="$1" if [ "$(_store_format)" = "2" ]; then @@ -2183,9 +2183,6 @@ _migrate_project() { info "Copy-forward for '$project': $moved new v2 twin(s), $already already present. v1 blobs kept (non-destructive). Run 'secrets migrate --finalize' once every project is migrated and every machine is upgraded." } -# Store-wide finalize: the only destructive step. Refuses unless verify --all -# is green and every v1 properties blob has a v2 twin. Cuts a recovery tag, -# stamps the marker, then drops v1 blobs. # Read-only survey: walk every project dir in the store and report each one's # v2-readiness from the blobs on disk (no manifest, no decryption). Exits # non-zero when any v1 properties blob lacks a v2 twin (i.e. the store is not @@ -2226,6 +2223,9 @@ _migrate_status() { return 0 } +# Store-wide finalize: the only destructive step. Refuses unless verify --all +# is green and every v1 properties blob has a v2 twin. Cuts a recovery tag, +# stamps the marker, then drops v1 blobs. _migrate_finalize() { local force="$1" check_key From 54575af61c7f0c1e170ac160ce01750ebfcb560c Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 09:45:10 -0700 Subject: [PATCH 33/70] docs: additive-v2 dual-write design spec (defuse the finalize gate) --- ...026-06-08-additive-v2-dual-write-design.md | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md diff --git a/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md b/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md new file mode 100644 index 0000000..7827e44 --- /dev/null +++ b/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md @@ -0,0 +1,203 @@ +# Additive v2 — automatic dual-write (no flag-day finalize) + +**Status:** Design approved (2026-06-08), pending implementation plan. +**Supersedes the framing of:** EGB-703 ("finalize is the destructive milestone you must reach") and EGB-709 ("delete v1 paths once every client is v2"). +**Relates to:** EGB-677 (storage-model unification), EGB-710 (manifest-free migrate). + +## Problem + +Store-format v2 (EGB-703) renamed the `properties` external blob suffix +(`.gradle-properties.age` → `.properties.age`) and made the transition a +three-step migration ending in a **destructive** `secrets migrate --finalize` +that drops the v1 blobs. Finalize is gated on "every machine and store in use is +v2" — a coordination requirement that's trivial for a solo dev but **uncertain +for a distributed team of 2–3+**: there is no reliable "all-clear" signal, and +finalizing early silently cuts an un-upgraded client off from `properties` +externals (it looks for `.gradle-properties.age`, which is gone). + +The hard gate exists **only** because finalize permanently drops the v1 blobs. +Remove the obligation to drop, and there is nothing to coordinate. + +## Decision + +Adopt **Strategy B — Additive v2**: "becoming v2" stops being a destructive +milestone and becomes a property an upgraded client maintains **automatically**. +Reads try both suffixes; writes keep alive whatever old clients already knew. +There is no required `migrate` ceremony and no required `finalize` — **upgrading +the `secrets` tool *is* the migration.** `finalize` survives only as optional, +indefinitely-deferrable garbage collection. + +Direction chosen over the alternatives: "park it / never finalize" (leaves the +two-format wart in place, resolves nothing) and "abandon the suffix rename" +(reverts shipped EGB-703 behavior, keeps the ugly `gradle-properties` suffix +forever). Additive v2 is the only option that reaches a clean v2 end-state +*without* a coordinated flag-day. + +## Design + +### 1. Read resolution — suffix-agnostic + +Any upgraded client resolving a `properties` external blob looks for the v2 +suffix `.properties.age` first, then falls back to the v1 `.gradle-properties.age`. +An upgraded client therefore **never fails to find a blob** regardless of which +suffix is on disk. Old (pre-0.6.0.0) clients still read v1-only — shipped code +cannot be changed. + +Code: today's `_external_blob_suffix(type)` (a single suffix string) is replaced +by a resolver that returns the **path of the blob that exists** for a given +`(project, slug, type)`, trying v2 then v1. `file` externals are unchanged in +both formats, so the resolver is a no-op identity for them (single suffix +`.file.age`). + +### 2. Write rule — the self-managing twin rule + +On push of a `properties` external, the client checks the store for an existing +v1 twin (`.gradle-properties.age`): + +- **v1 twin exists** (the external was first pushed by an old client) → + **dual-write both suffixes.** Old clients stay fresh forever; nobody is cut off. +- **No v1 twin exists** (brand-new external, first pushed by an upgraded client) → + **write v2-only (`.properties.age`).** Old clients cannot see it → the gentle, + *intended* forcing function. It only ever bites on genuinely new externals, + never on anything that previously worked. + +No stored state is required: the presence/absence of the v1 twin **is** the +signal. Mental model: *"keep alive what old clients already knew; new things are +v2-only."* + +### 3. The marker — auto-stamped, informational only + +The first push by an upgraded client stamps `.secrets-format = 2`. Because writes +are now driven by the twin rule (not the marker), the marker no longer decides +blob location — it becomes purely "a v2-aware client has touched this store." +`secrets which` continues to report `format: vN`. + +### 4. `finalize` → optional GC, never required + +`secrets migrate --finalize` remains the *only* operation that stops dual-writing +and drops the v1 twins. It is pure space reclamation (kilobytes), still +coordination-gated **if** you choose to run it, but never obligatory and +deferrable forever. Its existing safety posture is unchanged (recovery tag, +`verify --all` green gate, twin-before-drop, `--yes`/operator confirmation). This +is what defuses the gate: the destructive step still exists but is now optional +housekeeping, not a release blocker. + +### 5. `migrate` / `migrate --status` → normalize + inspect + +- `secrets migrate` (the EGB-710 manifest-free copy-forward) stays as an optional + "backfill v2 twins for existing v1-only externals" command — useful right + before a `finalize`. With read-fallback (§1) it is no longer required for + correctness, only for tidiness. +- `secrets migrate --status` becomes the **coverage survey**: per external, which + suffixes exist, and whether old clients are still being served (i.e. whether a + v1 twin is still present and being dual-written). This is the operator's + dashboard for "is anyone still relying on v1?" before an optional `finalize`. + +### 6. Propagation semantics (→ user docs verbatim) + +"Has not migrated" means **old client** (`secrets` < 0.6.0.0), not "hasn't run +`migrate`". `migrate` is a per-store op done once by anyone; what protects a +teammate is their **client version**. A read-only teammate needs only the tool +`git pull` (binary ≥ 0.6.0.0), not to run `migrate` themselves. "Upgrade your +secrets" (git pull the tool and/or migrate the store) is the correct umbrella — +and for a solo dev the two are one motion. + +Does a teammate on an old client get a secret User 1 just added? + +| Secret type | Blob name v1 vs v2 | Old client gets the new secret? | +|---|---|---| +| **dotenv** (`.env`, `.env.*`, `.dev.vars`) | identical (`.env.age`) | **Yes, always.** No forcing function possible — the blob name never changed. | +| **`file` external** (keystore, etc.) | identical (`.file.age`) | **Yes, always.** | +| **`properties` external** — existing (has a v1 twin) | dual-written | **Yes.** Old client reads the maintained `.gradle-properties.age`. | +| **`properties` external** — brand-new (no v1 twin) | v2-only | **No → must upgrade.** The forcing function; rare, and never breaks anything that previously worked. | + +Net: a teammate on an old client keeps getting **all** everyday `.env` updates +indefinitely (a good safety property — no one silently misses everyday secrets), +and only hits a wall on a genuinely new `properties`-style external. + +### 7. Known caveat (documented, not engineered around) + +Dual-write keeps v1 *readers* fresh, but a v1 *writer* writes only +`.gradle-properties.age`. A v2 reader (reading `.properties.age` first) could +therefore read stale data until that external is re-pushed by an upgraded client. +For the normal shape — one writer per external, who upgrades first — it never +bites. This matches today's reality and is documented rather than solved +(solving it would require timestamp/newest-wins arbitration across two encrypted +blobs — YAGNI for v1). + +### 8. Backward-compatibility matrix + +| Actor | dotenv / `file` | `properties` (existing twin) | `properties` (new, v2-only) | +|---|---|---|---| +| Upgraded client reads | ✓ | ✓ (resolver finds either) | ✓ | +| Upgraded client writes | unchanged | dual-writes both | writes v2-only | +| Old client reads | ✓ | ✓ (reads maintained v1 twin) | ✗ (forcing function) | +| Old client writes | unchanged | writes v1 twin only (see §7) | n/a (can't create v2) | + +## Code-level surface (for the implementation plan) + +- **`_external_blob_suffix(type)`** → split into: + - `_resolve_external_blob_read(project, slug, type)` — returns the path of the + blob that exists, trying `.properties.age` then `.gradle-properties.age` for + `properties`; identity for `file`. Used by `pull_external_files`, `verify`, + and any read path. + - `_external_blob_write_targets(project, slug, type)` — returns the suffix + path(s) to write: for `properties`, both suffixes when a v1 twin already + exists, else v2-only; single path for `file`. +- **`push_external_files`** — write to every path from + `_external_blob_write_targets` (was a single `age -o`); auto-stamp the marker + on first push by an upgraded client. +- **`pull_external_files` / `cmd_verify`** — resolve blobs via + `_resolve_external_blob_read` (was the single-suffix lookup). +- **`_migrate_finalize`** — unchanged logic; doc/help reframed as optional GC. +- **`_migrate_project` / `_migrate_status`** — retained (EGB-710); `--status` + extended to report dual-write coverage per external. +- **Docs:** CLAUDE.md "Store format" bullet, README, `cmd_help`, CHANGELOG, + VERSION bump (minor — new write semantics). + +## Safety / constraints (unchanged invariants) + +- bash 3.2 portable; every store walk stays recursive (`find -type f`). +- Path-validation rails (`_validate_external_target_path`, slug derivation, + symlink/`..` refusal) untouched. +- `finalize`'s destructive gating (recovery tag, verify-green, twin-before-drop, + `--yes`) untouched. +- The store still carries no manifest; `.secrets.json` remains per-project. + +## Test plan (bats, outline) + +1. Read-fallback: a v2 client resolves a `properties` external that exists only + as `.gradle-properties.age` (no twin) — pull succeeds. +2. Twin rule — existing twin → dual-write: push an external that has a v1 twin; + assert **both** suffixes are written and an old-client read path (v1 suffix) + sees the fresh value. +3. Twin rule — new external → v2-only: push a brand-new `properties` external; + assert **only** `.properties.age` is written (no v1 twin created) — the + forcing function. +4. Marker auto-stamp: first push by an upgraded client on a markerless store + stamps `.secrets-format = 2`. +5. dotenv/`file` unaffected: a dotenv and a `file` external round-trip identically + regardless of store marker. +6. `migrate --status` coverage: reports which externals are dual-written vs + v2-only. +7. `finalize` still green: existing finalize gates and drop behavior unchanged. +8. Caveat is observable (optional): a v1-suffix-only update is read by the v2 + client via fallback (documents the one-writer assumption). + +## Ripple to the roadmap + +- **EGB-703**: "finalize is the destructive milestone you must reach" → "finalize + is optional GC." Update the ticket/notes. +- **EGB-709** (collapse v1 paths): no longer gates on "all clients v2." Becomes + "delete the dual-write/transitional code **if/when** every store is GC'd to + pure v2" — much later, low stakes. Add a note to EGB-709. +- A new ticket should track this work (additive-v2 dual-write). + +## Decided knobs (no longer open) + +- Migration ceremony: **automatic** (upgraded client dual-writes on push; no + required `migrate`). `migrate`/`--status` remain manual/inspection tools. +- Forcing function: **kept**, scoped to brand-new `properties` externals via the + twin rule (existing twins always dual-written). +- `finalize`: **kept** as optional GC (not removed), so a fully-upgraded store + can still be reclaimed to pure v2. From ee4ea413ef0171f8880e6cac7d52f291e4e06287 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 09:58:23 -0700 Subject: [PATCH 34/70] =?UTF-8?q?docs:=20additive-v2=20implementation=20pl?= =?UTF-8?q?an=20+=20spec=20=C2=A73=20fix=20(no=20marker=20auto-stamp)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-06-08-additive-v2-dual-write.md | 587 ++++++++++++++++++ ...026-06-08-additive-v2-dual-write-design.md | 31 +- 2 files changed, 609 insertions(+), 9 deletions(-) create mode 100644 docs/superpowers/plans/2026-06-08-additive-v2-dual-write.md diff --git a/docs/superpowers/plans/2026-06-08-additive-v2-dual-write.md b/docs/superpowers/plans/2026-06-08-additive-v2-dual-write.md new file mode 100644 index 0000000..4b21c0c --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-additive-v2-dual-write.md @@ -0,0 +1,587 @@ +# Additive-v2 Dual-Write Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make store-format v2 *additive* — upgraded `secrets` clients read either blob suffix and dual-write existing `properties` externals — so the destructive `migrate --finalize` becomes optional GC and the cross-machine coordination gate disappears. + +**Architecture:** Replace the marker-driven single-suffix helper `_external_blob_suffix(type)` with two helpers: a read-resolver that tries `.properties.age` then falls back to `.gradle-properties.age`, and a write-targets helper that writes the v2 suffix always plus the v1 suffix *only when a v1 twin already exists* (dual-write existing externals; brand-new externals are v2-only — the intended forcing function). Reads and writes no longer depend on the `.secrets-format` marker, which keeps its meaning (born-v2 / finalized). Because fresh pushes now write the v2 suffix on any store, the migrate/finalize test fixtures (which relied on push producing a v1 blob) are updated to fabricate an old-client v1 blob. + +**Tech Stack:** Single bash 3.2 script (`secrets`); `age`, `git`, `jq`. Tests: `bats-core` (`test/migrate.bats`, `test/manifest.bats`). Every standalone `[[ ]]` test assertion ends with `|| false` (bash 3.2 ERR-trap gotcha). + +**Spec:** `docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md`. **Ticket:** EGB-712. + +--- + +## Background facts (verified against branch `brian/egb-712-...`, post-EGB-710) + +- `_external_blob_suffix()` — `secrets:548-555`. Returns `properties` for `gradle-properties` on a v2 store (`_store_format == 2`), else the type verbatim; `file` always returns `file`. Blob path = `external/..age`. +- Callers of `_external_blob_suffix`: push file write `secrets:655`, push properties write `secrets:684`, pull read `secrets:713`, verify read `secrets:2056`. All four are replaced; then the helper is deleted. +- `_secrets_files_slug(path)` (`secrets:512`) derives the machine-independent slug. dotenv and `file` blobs are byte-identical in v1/v2 (only the `properties` suffix diverges). +- `_store_format()` (`secrets:529`) and the marker stay as-is — set only by `init` (born-v2) and `migrate --finalize`. Push must NOT stamp it (see spec §3). +- Existing test that codifies OLD write behavior: `test/migrate.bats:52` "push on a v1 store still writes .gradle-properties.age (back-compat)" — rewritten in Task 2. +- Tests that fabricate-or-rely-on a v1 properties blob from `make_v1_store; push` and break once push writes v2-only (repaired in Task 3): the migrate copy-forward/idempotent tests, the `--dry-run` rename test, the dotenv+file untouched test, all four finalize tests, the EGB-710 manifest-free / undeclared-twin tests, and the two `--status` tests that need an actual v1 blob. (dotenv-only and already-v2 tests are unaffected.) +- Baseline before this plan: `bats test/` = 242 passing. + +## File structure + +- Modify: `secrets` — delete `_external_blob_suffix` (548-555); add `_resolve_external_blob_read` + `_external_blob_write_targets` in its place; rewire push (651-688), pull (712-713), verify (2052-2068); extend `_migrate_status`; docs in `cmd_help`. +- Modify: `test/migrate.bats` — add `m_fake_v1_blob` helper; add read-fallback + write-rule + status-coverage tests; repair the v1-blob-dependent fixtures. +- Modify: `CLAUDE.md`, `README.md`, `VERSION` (→ `0.7.0.0`), `CHANGELOG.md`. + +--- + +## Task 1: Read-resolver — reads try both suffixes + +**Files:** `secrets` (replace `_external_blob_suffix` with the read-resolver; rewire pull + verify reads), `test/migrate.bats`. + +- [ ] **Step 1: Write the failing test** — a v2 store whose properties blob exists ONLY in the v1 suffix must still pull. + +Add to `test/migrate.bats` (after the format-marker tests, ~line 60): + +```bash +@test "pull reads a v1-suffix properties blob on a v2 store (read-fallback)" { + init_with_remote # born-v2 store (marker=2) + m_gradle_src $'beaconClerkPkTest=pk_test_v1\n' + create_project_dir rffallback + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push rffallback >/dev/null 2>&1 # writes .properties.age on a v2 store + # Simulate an external that exists only in the v1 suffix (an old client wrote it): + local v2blob; v2blob=$(ls "$SECRETS_DIR/rffallback/external/"*.properties.age) + mv "$v2blob" "${v2blob%.properties.age}.gradle-properties.age" + rm -f "$HOME/.gradle/gradle.properties" + "$SECRETS_BIN" pull rffallback >/dev/null 2>&1 + run grep -q 'beaconClerkPkTest=pk_test_v1' "$HOME/.gradle/gradle.properties" + [ "$status" -eq 0 ] +} +``` + +- [ ] **Step 2: Run it, confirm it fails** + +Run: `bats test/migrate.bats -f "read-fallback"` +Expected: FAIL — old pull (`secrets:713`) uses `_external_blob_suffix gradle-properties` = `properties` on a v2 store, looks only for `.properties.age` (which we renamed away), warns "no encrypted data", restores nothing → the grep fails. + +- [ ] **Step 3: Replace `_external_blob_suffix` with the read-resolver.** Replace `secrets:542-555` (the comment block + `_external_blob_suffix()` through its closing `}`) with: + +```bash +# Resolve the on-disk path of an external blob for READING. Tries the v2 suffix +# (.properties.age) first, then falls back to the v1 (.gradle-properties.age) for +# `properties` externals, so an upgraded client finds the blob whichever format +# wrote it (additive v2 — EGB-712). `file` externals share one suffix in both +# formats. Echoes the path of the blob that exists; if neither exists, echoes the +# canonical v2 path so the caller's "no blob" message reads sensibly. Read-only. +_resolve_external_blob_read() { + local project="$1" slug="$2" mtype="$3" + local base="$SECRETS_DIR/$project/external/$slug" + case "$mtype" in + file) + echo "$base.file.age" ;; + properties|gradle-properties) + if [ -f "$base.properties.age" ]; then + echo "$base.properties.age" + elif [ -f "$base.gradle-properties.age" ]; then + echo "$base.gradle-properties.age" + else + echo "$base.properties.age" + fi ;; + *) + echo "$base.$mtype.age" ;; + esac +} +``` + +(The write-targets helper is added in Task 2 — leave a gap; do not reintroduce `_external_blob_suffix`.) + +- [ ] **Step 4: Rewire the pull read.** At `secrets:712-713`, replace: + +```bash + local slug; slug=$(_secrets_files_slug "$mpath") + local blob="$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" +``` + +with: + +```bash + local slug; slug=$(_secrets_files_slug "$mpath") + local blob; blob=$(_resolve_external_blob_read "$project" "$slug" "$mtype") +``` + +- [ ] **Step 5: Rewire the verify read.** At `secrets:2055-2065`, replace: + +```bash + slug=$(_secrets_files_slug "$epath") + erel="external/$slug.$(_external_blob_suffix "$etype").age" + # Account for BOTH the v1 and v2 suffix forms in the orphan set. During the + # migration window (after copy-forward, before --finalize) the v2 twin + # coexists with the v1 blob; neither should read as an orphan whichever + # format the store currently reports. (file's two forms are identical.) + expected="${expected}external/$slug.$etype.age"$'\n' + [ "$etype" = "gradle-properties" ] && expected="${expected}external/$slug.properties.age"$'\n' + eblob="$pdir/$erel" + if [ ! -f "$eblob" ]; then + echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 +``` + +with: + +```bash + slug=$(_secrets_files_slug "$epath") + # Account for BOTH suffix forms in the orphan set — a dual-written `properties` + # external (additive v2 — EGB-712) legitimately has both blobs on disk; neither + # is an orphan. (file's two forms are identical.) + expected="${expected}external/$slug.$etype.age"$'\n' + [ "$etype" = "gradle-properties" ] && expected="${expected}external/$slug.properties.age"$'\n' + eblob=$(_resolve_external_blob_read "$project" "$slug" "$etype") + erel="${eblob#"$pdir"/}" + if [ ! -f "$eblob" ]; then + echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 +``` + +(Note: `_external_blob_suffix` still has two remaining callers in push — push isn't rewired until Task 2, so the script still parses and runs. Those calls keep working because the function is only deleted in Task 2 Step 6, after push is rewired.) + +**IMPORTANT:** do NOT delete `_external_blob_suffix` yet — push (`secrets:655`, `secrets:684`) still calls it until Task 2. Deleting it now breaks push. + +- [ ] **Step 6: Run the read-fallback test + full migrate suite** + +Run: `bats test/migrate.bats` +Expected: the new "read-fallback" test PASSES; all other migrate tests still PASS (push unchanged; pull/verify now use the resolver, which is equivalent to the old behavior whenever the suffix matches the store format). + +- [ ] **Step 7: Commit** + +```bash +git add secrets test/migrate.bats +git commit -m "feat: read-resolver tries both external suffixes (additive v2, EGB-712)" +``` + +--- + +## Task 2: Write-targets — twin rule (dual-write existing, v2-only for new) + +**Files:** `secrets` (add `_external_blob_write_targets`; rewire push file + properties writes; delete `_external_blob_suffix`), `test/migrate.bats`. + +- [ ] **Step 1: Write the failing tests.** Add to `test/migrate.bats`: + +```bash +@test "push writes the v2 suffix for a fresh external even on a v1 store" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir freshv1 + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push freshv1 >/dev/null 2>&1 + run bash -c "ls $SECRETS_DIR/freshv1/external/*.properties.age" + [ "$status" -eq 0 ] # v2 suffix regardless of marker + run bash -c "ls $SECRETS_DIR/freshv1/external/*.gradle-properties.age 2>/dev/null" + [ "$status" -ne 0 ] # no v1 twin for a brand-new external +} + +@test "push dual-writes the v1 twin so old clients stay fresh" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_old\n' + create_project_dir dualwrite + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push dualwrite >/dev/null 2>&1 # v2-only (fresh) + m_fake_v1_twin dualwrite # simulate a pre-existing v1 twin + m_gradle_src $'beaconClerkPkTest=pk_test_new\n' + "$SECRETS_BIN" push dualwrite >/dev/null 2>&1 # twin exists -> dual-write both + # Prove the v1 twin was refreshed: drop the v2 blob, pull, expect the NEW value. + rm -f "$SECRETS_DIR/dualwrite/external/"*.properties.age + rm -f "$HOME/.gradle/gradle.properties" + "$SECRETS_BIN" pull dualwrite >/dev/null 2>&1 + run grep -q 'beaconClerkPkTest=pk_test_new' "$HOME/.gradle/gradle.properties" + [ "$status" -eq 0 ] +} +``` + +- [ ] **Step 2: Add the `m_fake_v1_twin` test helper.** In `test/migrate.bats`, next to `make_v1_store` (~line 12), add: + +```bash +# Simulate an old (v1) client's properties blob: copy the pushed v2 +# .properties.age to its v1 .gradle-properties.age twin. (Current clients never +# write the v1 suffix for a fresh external, so tests fabricate it.) Use `cp` to +# KEEP the v2 blob (dual present); see m_make_v1_only to leave only the v1 blob. +m_fake_v1_twin() { + local proj="$1" v2 + v2=$(ls "$SECRETS_DIR/$proj/external/"*.properties.age) + cp "$v2" "${v2%.properties.age}.gradle-properties.age" +} +# Like m_fake_v1_twin but renames (leaves ONLY the v1 blob) — for old-client-only +# / copy-forward fixtures. +m_make_v1_only() { + local proj="$1" v2 + v2=$(ls "$SECRETS_DIR/$proj/external/"*.properties.age) + mv "$v2" "${v2%.properties.age}.gradle-properties.age" +} +``` + +- [ ] **Step 3: Run the new tests, confirm they fail** + +Run: `bats test/migrate.bats -f "fresh external even on a v1 store"` +Expected: FAIL — on a v1 store, the old push (`_external_blob_suffix gradle-properties` = `gradle-properties`) writes `.gradle-properties.age`, so the `*.properties.age` assertion fails. +Run: `bats test/migrate.bats -f "dual-writes the v1 twin"` +Expected: FAIL — old push writes a single suffix; the second push won't refresh the v1 twin. + +- [ ] **Step 4: Add the write-targets helper.** Immediately after `_resolve_external_blob_read` (added in Task 1), insert: + +```bash +# The on-disk path(s) to WRITE for an external blob, one per line. For a +# `properties` external this is the v2 suffix (.properties.age) ALWAYS, plus the +# v1 suffix (.gradle-properties.age) WHEN a v1 twin already exists in the store +# (dual-write keeps old clients fresh; a brand-new external is v2-only — the +# intended forcing function, additive v2 / EGB-712). `file` externals have a +# single suffix in both formats. Independent of the store marker. +_external_blob_write_targets() { + local project="$1" slug="$2" mtype="$3" + local base="$SECRETS_DIR/$project/external/$slug" + case "$mtype" in + file) + echo "$base.file.age" ;; + properties|gradle-properties) + echo "$base.properties.age" + [ -f "$base.gradle-properties.age" ] && echo "$base.gradle-properties.age" ;; + *) + echo "$base.$mtype.age" ;; + esac +} +``` + +- [ ] **Step 5: Rewire the push writes.** At `secrets:651-658` (the `file` branch), replace: + +```bash + if [ "$mtype" = "file" ]; then + # EGB-652: whole-file sync — encrypt the file verbatim (binary-safe). + mkdir -p "$SECRETS_DIR/$project/external" + local fslug; fslug=$(_secrets_files_slug "$mpath") + age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.$(_external_blob_suffix file).age" "$expanded" + info "Encrypted file $mpath" + pushed=$((pushed + 1)) + continue + fi +``` + +with: + +```bash + if [ "$mtype" = "file" ]; then + # EGB-652: whole-file sync — encrypt the file verbatim (binary-safe). + mkdir -p "$SECRETS_DIR/$project/external" + local fslug; fslug=$(_secrets_files_slug "$mpath") + local wt + while IFS= read -r wt; do + [ -n "$wt" ] || continue + age -r "$pubkey" -o "$wt" "$expanded" + done < <(_external_blob_write_targets "$project" "$fslug" file) + info "Encrypted file $mpath" + pushed=$((pushed + 1)) + continue + fi +``` + +Then at `secrets:682-684` (the properties branch), replace: + +```bash + mkdir -p "$SECRETS_DIR/$project/external" + local slug; slug=$(_secrets_files_slug "$mpath") + age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" "$tmp" +``` + +with: + +```bash + mkdir -p "$SECRETS_DIR/$project/external" + local slug; slug=$(_secrets_files_slug "$mpath") + local wt + while IFS= read -r wt; do + [ -n "$wt" ] || continue + age -r "$pubkey" -o "$wt" "$tmp" + done < <(_external_blob_write_targets "$project" "$slug" "$mtype") +``` + +- [ ] **Step 6: Delete the now-unused `_external_blob_suffix`.** Confirm zero remaining callers first: + +Run: `grep -n "_external_blob_suffix" secrets` +Expected: no matches (all four call sites rewired). If any remain, rewire them before deleting. Then delete the `_resolve_external_blob_read`-replaced... — it's already gone (replaced in Task 1). Verify the function is absent: `grep -c "_external_blob_suffix()" secrets` → `0`. + +- [ ] **Step 7: Run the two new write tests** + +Run: `bats test/migrate.bats -f "fresh external even on a v1 store"` then `-f "dual-writes the v1 twin"` +Expected: both PASS. + +- [ ] **Step 8: Rewrite the obsolete back-compat test.** Replace the `@test "push on a v1 store still writes .gradle-properties.age (back-compat)"` block (`test/migrate.bats:52-60`) with: + +```bash +@test "push on a v1 store writes the v2 suffix for a fresh external (additive v2)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir v1push + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push v1push >/dev/null 2>&1 + run bash -c "ls $SECRETS_DIR/v1push/external/*.properties.age" + [ "$status" -eq 0 ] +} +``` + +- [ ] **Step 9: Run the full migrate suite — expect the v1-blob-dependent fixtures to FAIL.** This is expected; Task 3 repairs them. + +Run: `bats test/migrate.bats` +Expected: the read-fallback + two write tests + rewritten back-compat test PASS; several copy-forward/finalize/status tests now FAIL (push no longer writes a `.gradle-properties.age` for them to migrate). Note which fail — Task 3 fixes exactly those. + +- [ ] **Step 10: Commit** (suite intentionally not yet fully green — Task 3 follows immediately) + +```bash +git add secrets test/migrate.bats +git commit -m "feat: twin-rule write targets — dual-write existing, v2-only for new (additive v2, EGB-712)" +``` + +--- + +## Task 3: Repair migrate/finalize/status fixtures + +**Files:** `test/migrate.bats`. No production code changes — this re-greens the suite by fabricating the old-client v1 blobs that push no longer writes. + +The rule for each repair: after the `"$SECRETS_BIN" push ` line, insert a fabrication call: +- Use **`m_make_v1_only `** (rename → only the v1 blob exists) for tests asserting a `*.gradle-properties.age` blob exists / is copy-forwarded (mirrors the pre-EGB-712 state where push produced a v1 blob). +- Use **`m_fake_v1_twin `** (keep both) only where a test needs both suffixes present. + +- [ ] **Step 1: Repair the copy-forward / dry-run / idempotent tests.** In each of these tests, insert `m_make_v1_only ` immediately after the `push ` line: + - `"migrate --dry-run reports the rename and writes nothing"` (proj `dryproj`) + - `"migrate copy-forward creates the v2 twin and keeps the v1 blob (byte-identical)"` (proj `cfproj`) + - `"migrate copy-forward is idempotent"` (proj `idemproj`) + - `"migrate leaves dotenv and file blobs untouched"` (proj `mixproj`) — note this one ALSO pushes a `file` external; `m_make_v1_only` only touches `*.properties.age`, leaving the `.file.age` blob alone (correct). + +Worked example — the copy-forward test becomes: + +```bash +@test "migrate copy-forward creates the v2 twin and keeps the v1 blob (byte-identical)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir cfproj + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push cfproj >/dev/null 2>&1 + m_make_v1_only cfproj + local old; old=$(ls "$SECRETS_DIR/cfproj/external/"*.gradle-properties.age) + run "$SECRETS_BIN" migrate + [ "$status" -eq 0 ] + local new; new=$(ls "$SECRETS_DIR/cfproj/external/"*.properties.age) + [ -f "$old" ] + [ -f "$new" ] + cmp -s "$old" "$new" +} +``` + +- [ ] **Step 2: Repair the finalize tests.** Insert `m_make_v1_only ` after the `push ` line in: + - `"finalize refuses when verify --all is not green"` (proj `failverify`) — then the existing `migrate` step creates the twin; corrupting `*.properties.age` still trips verify. + - `"finalize refuses an un-twinned v1 blob (project not migrated)"` (proj `untwinned`) — leaves a lone v1 blob, no twin: exactly the un-twinned state the test wants. + - `"finalize green path drops v1, keeps v2, stamps the marker"` (proj `finproj`). + - `"finalize cuts a recovery tag before deleting v1 blobs"` (proj — read it from the test). + +- [ ] **Step 3: Repair the EGB-710 manifest-free / undeclared-twin tests.** Insert `m_make_v1_only ` after the `push ` line in: + - `"migrate copy-forwards a v1 properties blob with no .secrets.json (manifest-free)"` (proj `nomanifestblob`) — insert BEFORE the `rm -f .secrets.json` line. + - `"migrate twins a store blob even when the manifest no longer declares it"` (proj `staleblob`) — insert before the `.secrets.json` rewrite. + +- [ ] **Step 4: Repair the `--status` tests that need a real v1 blob.** + - `"migrate --status flags a project that needs migrating"` (proj `needsmig`) — insert `m_make_v1_only needsmig` after push, so a lone un-twinned v1 blob exists → NEEDS MIGRATE. + - `"migrate --status reports finalize-ready once every blob is twinned"` (proj `readymig`) — insert `m_make_v1_only readymig` after push and BEFORE the `migrate` step (migrate then creates the twin → finalize-ready). + +- [ ] **Step 5: Run the full migrate suite** + +Run: `bats test/migrate.bats` +Expected: ALL pass. If any copy-forward test still reports "nothing to migrate", its `m_make_v1_only` call is missing or misplaced (must come after push, before migrate). + +- [ ] **Step 6: Run the WHOLE suite** (manifest.bats exercises externals end-to-end and must still be green) + +Run: `bats test/` +Expected: all pass. If a `manifest.bats` external test fails, check it isn't asserting a specific suffix that additive-v2 changed (a fresh push now writes `.properties.age`); update such an assertion the same way (assert `.properties.age`, or use the resolver-agnostic round-trip via pull). + +- [ ] **Step 7: Commit** + +```bash +git add test/migrate.bats +git commit -m "test: fabricate old-client v1 blobs in migrate/finalize/status fixtures (additive v2, EGB-712)" +``` + +--- + +## Task 4: `migrate --status` — dual-write coverage line + +**Files:** `secrets` (`_migrate_status`, `secrets:2190-2227`), `test/migrate.bats`. + +Adds visibility into which `properties` externals are v2-only (old clients can't read them — the forcing function) vs dual-written (old clients still served). + +- [ ] **Step 1: Write the failing test.** Add to `test/migrate.bats`: + +```bash +@test "migrate --status counts v2-only externals (old clients not served)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir v2onlyext + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push v2onlyext >/dev/null 2>&1 # v2-only (fresh, no v1 twin) + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] # no v1 blobs -> finalize-ready + [[ "$output" == *"v2-only"* ]] || false # surfaced as v2-only coverage +} +``` + +- [ ] **Step 2: Run it, confirm it fails** + +Run: `bats test/migrate.bats -f "counts v2-only externals"` +Expected: FAIL — `_migrate_status` currently only counts `*.gradle-properties.age`; it never mentions `v2-only`. + +- [ ] **Step 3: Extend `_migrate_status`.** In `_migrate_status` (`secrets:2190`), inside the `for dir` loop, after the existing `while ... done < <(find "$dir" -type f -name '*.gradle-properties.age' ...)` block and before the per-project classification, add a v2-only count, then surface it in the per-project line. Concretely, replace the classification block: + +```bash + if [ "$v1" -eq 0 ]; then + echo " $project: v2-ready (no v1 properties blobs)" + elif [ "$untwinned" -eq 0 ]; then + echo " $project: migrated ($v1 v1 blob(s), all twinned)" + else + echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'" + any_untwinned=1 + fi +``` + +with: + +```bash + # v2-only externals: a .properties.age with no .gradle-properties.age twin — + # old (v1) clients cannot read these (the additive-v2 forcing function). + local v2only=0 pf + while IFS= read -r pf; do + [ -f "$pf" ] || continue + [ -f "${pf%.properties.age}.gradle-properties.age" ] || v2only=$((v2only + 1)) + done < <(find "$dir" -type f -name '*.properties.age' 2>/dev/null) + local v2note="" + [ "$v2only" -gt 0 ] && v2note=" [$v2only v2-only — old clients not served]" + if [ "$v1" -eq 0 ]; then + echo " $project: v2-ready (no v1 properties blobs)$v2note" + elif [ "$untwinned" -eq 0 ]; then + echo " $project: migrated ($v1 v1 blob(s), all twinned)$v2note" + else + echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'$v2note" + any_untwinned=1 + fi +``` + +(`local` inside the loop is bash-3.2-fine — it re-declares per iteration.) + +- [ ] **Step 4: Run the new test + status suite** + +Run: `bats test/migrate.bats -f "status"` +Expected: all status tests PASS, including the new v2-only one. + +- [ ] **Step 5: Run the full suite** + +Run: `bats test/` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add secrets test/migrate.bats +git commit -m "feat: migrate --status surfaces v2-only externals (coverage, EGB-712)" +``` + +--- + +## Task 5: Docs, help, version, changelog + +**Files:** `secrets` (`cmd_help`), `CLAUDE.md`, `README.md`, `VERSION`, `CHANGELOG.md`. + +- [ ] **Step 1: `cmd_help` — reframe finalize as optional.** In `cmd_help()`, replace the finalize line: + +``` + secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) +``` + +with: + +``` + secrets migrate --finalize Optional GC: drop v1 blobs and mark the store pure v2 +``` + +- [ ] **Step 2: `CLAUDE.md` — additive-v2 paragraph.** In the "Store format" bullet (line ~65), after the migration-chain sentence, add (new sentence, same bullet): + +``` + Additive v2 (EGB-712): upgraded clients read either blob suffix (`_resolve_external_blob_read` tries `.properties.age` then `.gradle-properties.age`) and dual-write a `properties` external only when a v1 twin already exists (`_external_blob_write_targets`) — so existing externals keep old clients fresh, brand-new externals are v2-only (a gentle forcing function), and `migrate --finalize` is now OPTIONAL GC rather than a required, coordination-gated flag-day. dotenv and `file` blobs are identical across formats, so they always propagate to old clients. +``` + +- [ ] **Step 3: `README.md` — propagation table.** Add, near the migrate rows in the command table or in a short "Upgrading / store format" subsection, the propagation-by-secret-type summary (verbatim from spec §6): + +```markdown +**Do teammates on an older `secrets` get new secrets?** + +| Secret type | Old client gets it? | +|---|---| +| `.env` / `.env.*` / `.dev.vars` | **Yes, always** (blob name identical across formats) | +| whole-file external | **Yes, always** | +| `properties` external that already existed | **Yes** (dual-written) | +| brand-new `properties` external | **No — must upgrade `secrets`** (the forcing function) | + +"Upgrade your secrets" = `git pull` the tool clone (binary ≥ 0.6.0.0) and/or `secrets migrate` the store. A read-only teammate only needs the tool `git pull`. +``` + +- [ ] **Step 4: `VERSION`** — set to `0.7.0.0`. + +- [ ] **Step 5: `CHANGELOG.md`** — insert above the top entry: + +```markdown +## [0.7.0.0] - 2026-06-08 + +### Changed + +- **Additive store-format v2 (EGB-712)** — upgraded `secrets` clients now read + either external blob suffix (`.properties.age` or the legacy + `.gradle-properties.age`) and **dual-write** a `properties` external whenever a + v1 twin already exists in the store. Existing externals keep working for + teammates on an older `secrets`; only a brand-new `properties` external is + written v2-only (a gentle "upgrade to see it" forcing function). dotenv and + whole-`file` externals are unchanged across formats and always propagate. +- **`secrets migrate --finalize` is now optional GC**, not a required milestone. + Because clients dual-write and read-fall-back, no teammate is ever cut off by + *not* finalizing; finalize only reclaims the duplicate v1 blobs, and stays + deferrable indefinitely. Its safety gates are unchanged. + +### Added + +- **`secrets migrate --status`** now reports `v2-only` externals per project + (the ones an un-upgraded client cannot read), so you can see the forcing + function's footprint at a glance. +``` + +- [ ] **Step 6: Run the full suite** + +Run: `bats test/` +Expected: all pass (docs don't affect tests). Confirm the count is baseline 242 + net new tests from Tasks 1/2/4 (read-fallback, two write tests, v2-only status) minus the rewritten back-compat test (replaced, not added) = **246**. + +- [ ] **Step 7: Sanity — help renders** + +Run: `./secrets help 2>&1 | grep -- "--finalize"` +Expected: shows the reframed "Optional GC" line. (Help only prints; touches no store.) + +- [ ] **Step 8: Commit** + +```bash +git add secrets CLAUDE.md README.md VERSION CHANGELOG.md +git commit -m "docs: additive-v2 propagation + optional-GC finalize; bump 0.7.0.0 (EGB-712)" +``` + +--- + +## Self-review against the spec + +- **§1 read resolution** → Task 1 (`_resolve_external_blob_read`, wired into pull + verify). Test: read-fallback. +- **§2 write rule / twin rule** → Task 2 (`_external_blob_write_targets`, push wiring). Tests: fresh-→v2-only, existing-twin-→dual-write. +- **§3 marker NOT stamped** → no production change (push never touches the marker); guarded implicitly by Task 3 keeping the `make_v1_store; push; migrate` flow working (store stays markerless after push). The plan deliberately does not add marker-stamping. +- **§4 finalize = optional GC** → unchanged logic; reframed in Task 5 docs/help. Existing finalize tests stay green (Task 3 keeps them green). +- **§5 migrate / --status coverage** → Task 4 (v2-only line); `migrate` copy-forward unchanged (EGB-710). +- **§6 propagation table** → Task 5 README/CLAUDE.md. +- **§7 caveat** → documented in CHANGELOG/CLAUDE.md framing; the read-fallback test exercises the v1-only read path. +- **§8 back-compat matrix** → covered across Task 1 (reads), Task 2 (writes), Task 3 (old-client v1 blobs simulated). +- **Safety invariants** → no path-rail changes; bash 3.2 (no associative arrays; `while read` + `find`); recursive walks unchanged; finalize gating untouched. + +**Type/name consistency:** `_resolve_external_blob_read(project, slug, mtype)` and `_external_blob_write_targets(project, slug, mtype)` use the same arg order everywhere; test helpers `m_fake_v1_twin` (keep both) and `m_make_v1_only` (rename to v1-only) are used consistently per their documented semantics. + +**Placeholder scan:** none — every step carries verbatim code or an exact enumerated edit with the precise insertion point. + +## Operator-local follow-up (not part of this plan) + +Per `.ship-policy.json`, before any PR ask the operator to run `./test/run-security.sh` and complete the SIGNOFF. EGB-712 also needs its one stale AC bullet ("Marker auto-stamps…") corrected to match the §3 decision (no auto-stamp) — a one-line Linear edit. diff --git a/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md b/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md index 7827e44..cda4e6d 100644 --- a/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md +++ b/docs/superpowers/specs/2026-06-08-additive-v2-dual-write-design.md @@ -65,12 +65,24 @@ No stored state is required: the presence/absence of the v1 twin **is** the signal. Mental model: *"keep alive what old clients already knew; new things are v2-only."* -### 3. The marker — auto-stamped, informational only +### 3. The marker — unchanged meaning, NOT auto-stamped -The first push by an upgraded client stamps `.secrets-format = 2`. Because writes -are now driven by the twin rule (not the marker), the marker no longer decides -blob location — it becomes purely "a v2-aware client has touched this store." -`secrets which` continues to report `format: vN`. +Additive-v2 read/write behavior does **not** depend on the `.secrets-format` +marker at all (reads try both suffixes; writes follow the twin rule). So the +marker is left exactly as it is today: set only by `init` (born-v2, a fresh pure-v2 +store) or `migrate --finalize` (a store GC'd to pure v2). A transitional store +that upgraded clients are dual-writing stays **markerless (v1)** — which is +*accurate*: it has not been finalized to pure v2, and v1 twins still exist. + +Push deliberately does **not** auto-stamp the marker. (An earlier draft proposed +auto-stamping on first push; that was dropped during planning because it would +make a v1 store read as v2 the moment anyone pushed — turning every `migrate` +into a no-op and contradicting the "v1 until finalized" model the whole +migration relies on. The marker's only purposes — `secrets which` display and +gating `migrate`/`finalize` — are better served by it continuing to mean +"finalized/pure v2.") + +`secrets which` continues to report `format: vN` from the marker. ### 4. `finalize` → optional GC, never required @@ -145,8 +157,8 @@ blobs — YAGNI for v1). path(s) to write: for `properties`, both suffixes when a v1 twin already exists, else v2-only; single path for `file`. - **`push_external_files`** — write to every path from - `_external_blob_write_targets` (was a single `age -o`); auto-stamp the marker - on first push by an upgraded client. + `_external_blob_write_targets` (was a single `age -o`). Push does **not** stamp + the marker (see §3). - **`pull_external_files` / `cmd_verify`** — resolve blobs via `_resolve_external_blob_read` (was the single-suffix lookup). - **`_migrate_finalize`** — unchanged logic; doc/help reframed as optional GC. @@ -174,8 +186,9 @@ blobs — YAGNI for v1). 3. Twin rule — new external → v2-only: push a brand-new `properties` external; assert **only** `.properties.age` is written (no v1 twin created) — the forcing function. -4. Marker auto-stamp: first push by an upgraded client on a markerless store - stamps `.secrets-format = 2`. +4. Marker NOT stamped on push: a `make_v1_store` + push leaves `.secrets-format` + absent (store stays v1/transitional); the existing `make_v1_store; push; + migrate` flow is unaffected. 5. dotenv/`file` unaffected: a dotenv and a `file` external round-trip identically regardless of store marker. 6. `migrate --status` coverage: reports which externals are dual-written vs From 2866e5f4b1eccd45e079485af90d929e81b2eb38 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 10:08:41 -0700 Subject: [PATCH 35/70] feat: read-resolver tries both external suffixes (additive v2, EGB-712) --- secrets | 38 +++++++++++++++++++++++++++++++------- test/migrate.bats | 15 +++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/secrets b/secrets index 02a3415..5b41f03 100755 --- a/secrets +++ b/secrets @@ -554,6 +554,31 @@ _external_blob_suffix() { fi } +# Resolve the on-disk path of an external blob for READING. Tries the v2 suffix +# (.properties.age) first, then falls back to the v1 (.gradle-properties.age) for +# `properties` externals, so an upgraded client finds the blob whichever format +# wrote it (additive v2 — EGB-712). `file` externals share one suffix in both +# formats. Echoes the path of the blob that exists; if neither exists, echoes the +# canonical v2 path so the caller's "no blob" message reads sensibly. Read-only. +_resolve_external_blob_read() { + local project="$1" slug="$2" mtype="$3" + local base="$SECRETS_DIR/$project/external/$slug" + case "$mtype" in + file) + echo "$base.file.age" ;; + properties|gradle-properties) + if [ -f "$base.properties.age" ]; then + echo "$base.properties.age" + elif [ -f "$base.gradle-properties.age" ]; then + echo "$base.gradle-properties.age" + else + echo "$base.properties.age" + fi ;; + *) + echo "$base.$mtype.age" ;; + esac +} + # Merge managed key=value lines (from $2) into target file $1, preserving # all unrelated lines/comments/order. Updates a managed key in place (first # occurrence), collapses duplicates, appends new keys. Atomic + mode-safe. @@ -710,7 +735,7 @@ pull_external_files() { continue fi local slug; slug=$(_secrets_files_slug "$mpath") - local blob="$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" + local blob; blob=$(_resolve_external_blob_read "$project" "$slug" "$mtype") if [ ! -f "$blob" ]; then echo "WARNING: $SECRETS_FILES_NAME names '$mpath' but no encrypted data exists in the store yet. Run 'secrets push' on a machine that has these keys. Skipping." >&2 continue @@ -2053,14 +2078,13 @@ _verify_project() { while IFS=$'\t' read -r etype epath _; do [ -n "$etype" ] || continue slug=$(_secrets_files_slug "$epath") - erel="external/$slug.$(_external_blob_suffix "$etype").age" - # Account for BOTH the v1 and v2 suffix forms in the orphan set. During the - # migration window (after copy-forward, before --finalize) the v2 twin - # coexists with the v1 blob; neither should read as an orphan whichever - # format the store currently reports. (file's two forms are identical.) + # Account for BOTH suffix forms in the orphan set — a dual-written `properties` + # external (additive v2 — EGB-712) legitimately has both blobs on disk; neither + # is an orphan. (file's two forms are identical.) expected="${expected}external/$slug.$etype.age"$'\n' [ "$etype" = "gradle-properties" ] && expected="${expected}external/$slug.properties.age"$'\n' - eblob="$pdir/$erel" + eblob=$(_resolve_external_blob_read "$project" "$slug" "$etype") + erel="${eblob#"$pdir"/}" if [ ! -f "$eblob" ]; then echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 findings=$((findings + 1)) diff --git a/test/migrate.bats b/test/migrate.bats index 9441bca..e4a9091 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -67,6 +67,21 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -eq 0 ] } +@test "pull reads a v1-suffix properties blob on a v2 store (read-fallback)" { + init_with_remote # born-v2 store (marker=2) + m_gradle_src $'beaconClerkPkTest=pk_test_v1\n' + create_project_dir rffallback + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push rffallback >/dev/null 2>&1 # writes .properties.age on a v2 store + # Simulate an external that exists only in the v1 suffix (an old client wrote it): + local v2blob; v2blob=$(ls "$SECRETS_DIR/rffallback/external/"*.properties.age) + mv "$v2blob" "${v2blob%.properties.age}.gradle-properties.age" + rm -f "$HOME/.gradle/gradle.properties" + "$SECRETS_BIN" pull rffallback >/dev/null 2>&1 + run grep -q 'beaconClerkPkTest=pk_test_v1' "$HOME/.gradle/gradle.properties" + [ "$status" -eq 0 ] +} + # ─── migrate --dry-run / copy-forward (increment 2) ─────────────────── @test "migrate --dry-run reports the rename and writes nothing" { From 2f36fe18988702460d28a31b3ae6fe8df1d56c6a Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 10:31:00 -0700 Subject: [PATCH 36/70] =?UTF-8?q?feat:=20twin-rule=20write=20targets=20?= =?UTF-8?q?=E2=80=94=20dual-write=20existing,=20v2-only=20for=20new=20(add?= =?UTF-8?q?itive=20v2,=20EGB-712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- secrets | 47 ++++++++++++++++++++++++++++++----------------- test/migrate.bats | 45 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/secrets b/secrets index 5b41f03..ed6e1f4 100755 --- a/secrets +++ b/secrets @@ -539,21 +539,6 @@ _store_format() { echo 1 } -# The on-disk blob suffix for an external entry, format-aware. v2 unifies -# the legacy `gradle-properties` suffix to `properties` (matching the JSON -# manifest `type`); `file` is unchanged in both formats. The slug + this -# suffix + `.age` is the external blob name. This is the single source of -# truth for the suffix — push, pull, verify all route through it so a v1 -# and a v2 store can never disagree on where a blob lives. -_external_blob_suffix() { - local mtype="$1" - if [ "$mtype" = "gradle-properties" ] && [ "$(_store_format)" = "2" ]; then - echo "properties" - else - echo "$mtype" - fi -} - # Resolve the on-disk path of an external blob for READING. Tries the v2 suffix # (.properties.age) first, then falls back to the v1 (.gradle-properties.age) for # `properties` externals, so an upgraded client finds the blob whichever format @@ -579,6 +564,26 @@ _resolve_external_blob_read() { esac } +# The on-disk path(s) to WRITE for an external blob, one per line. For a +# `properties` external this is the v2 suffix (.properties.age) ALWAYS, plus the +# v1 suffix (.gradle-properties.age) WHEN a v1 twin already exists in the store +# (dual-write keeps old clients fresh; a brand-new external is v2-only — the +# intended forcing function, additive v2 / EGB-712). `file` externals have a +# single suffix in both formats. Independent of the store marker. +_external_blob_write_targets() { + local project="$1" slug="$2" mtype="$3" + local base="$SECRETS_DIR/$project/external/$slug" + case "$mtype" in + file) + echo "$base.file.age" ;; + properties|gradle-properties) + echo "$base.properties.age" + [ -f "$base.gradle-properties.age" ] && echo "$base.gradle-properties.age" ;; + *) + echo "$base.$mtype.age" ;; + esac +} + # Merge managed key=value lines (from $2) into target file $1, preserving # all unrelated lines/comments/order. Updates a managed key in place (first # occurrence), collapses duplicates, appends new keys. Atomic + mode-safe. @@ -677,7 +682,11 @@ push_external_files() { # EGB-652: whole-file sync — encrypt the file verbatim (binary-safe). mkdir -p "$SECRETS_DIR/$project/external" local fslug; fslug=$(_secrets_files_slug "$mpath") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.$(_external_blob_suffix file).age" "$expanded" + local wt + while IFS= read -r wt; do + [ -n "$wt" ] || continue + age -r "$pubkey" -o "$wt" "$expanded" + done < <(_external_blob_write_targets "$project" "$fslug" file) info "Encrypted file $mpath" pushed=$((pushed + 1)) continue @@ -706,7 +715,11 @@ push_external_files() { fi mkdir -p "$SECRETS_DIR/$project/external" local slug; slug=$(_secrets_files_slug "$mpath") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" "$tmp" + local wt + while IFS= read -r wt; do + [ -n "$wt" ] || continue + age -r "$pubkey" -o "$wt" "$tmp" + done < <(_external_blob_write_targets "$project" "$slug" "$mtype") rm -f "$tmp" info "Extracted $found key(s) from $mpath" pushed=$((pushed + 1)) diff --git a/test/migrate.bats b/test/migrate.bats index e4a9091..92e113c 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -10,6 +10,19 @@ make_v1_store() { init_with_remote rm -f "$SECRETS_DIR/.secrets-format" } +# Simulate an old (v1) client's properties blob: copy the pushed v2 +# .properties.age to its v1 .gradle-properties.age twin (KEEPS both present). +m_fake_v1_twin() { + local proj="$1" v2 + v2=$(ls "$SECRETS_DIR/$proj/external/"*.properties.age) + cp "$v2" "${v2%.properties.age}.gradle-properties.age" +} +# Like m_fake_v1_twin but RENAMES (leaves ONLY the v1 blob) — for copy-forward fixtures. +m_make_v1_only() { + local proj="$1" v2 + v2=$(ls "$SECRETS_DIR/$proj/external/"*.properties.age) + mv "$v2" "${v2%.properties.age}.gradle-properties.age" +} m_gradle_src() { mkdir -p "$HOME/.gradle"; printf '%s' "$1" > "$HOME/.gradle/gradle.properties"; } m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > "$HOME/keystores/upload.keystore"; } @@ -49,13 +62,41 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -ne 0 ] } -@test "push on a v1 store still writes .gradle-properties.age (back-compat)" { +@test "push on a v1 store writes the v2 suffix for a fresh external (additive v2)" { make_v1_store m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' create_project_dir v1push printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push v1push >/dev/null 2>&1 - run bash -c "ls $SECRETS_DIR/v1push/external/*.gradle-properties.age" + run bash -c "ls $SECRETS_DIR/v1push/external/*.properties.age" + [ "$status" -eq 0 ] +} + +@test "push writes the v2 suffix for a fresh external even on a v1 store" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir freshv1 + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push freshv1 >/dev/null 2>&1 + run bash -c "ls $SECRETS_DIR/freshv1/external/*.properties.age" + [ "$status" -eq 0 ] + run bash -c "ls $SECRETS_DIR/freshv1/external/*.gradle-properties.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "push dual-writes the v1 twin so old clients stay fresh" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_old\n' + create_project_dir dualwrite + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push dualwrite >/dev/null 2>&1 + m_fake_v1_twin dualwrite + m_gradle_src $'beaconClerkPkTest=pk_test_new\n' + "$SECRETS_BIN" push dualwrite >/dev/null 2>&1 + rm -f "$SECRETS_DIR/dualwrite/external/"*.properties.age + rm -f "$HOME/.gradle/gradle.properties" + "$SECRETS_BIN" pull dualwrite >/dev/null 2>&1 + run grep -q 'beaconClerkPkTest=pk_test_new' "$HOME/.gradle/gradle.properties" [ "$status" -eq 0 ] } From 69ab9636e94ad618af5164becad53dadf3aa703b Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 10:38:18 -0700 Subject: [PATCH 37/70] test: fabricate old-client v1 blobs in migrate/finalize/status fixtures (additive v2, EGB-712) --- test/migrate.bats | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/migrate.bats b/test/migrate.bats index 92e113c..281a281 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -131,6 +131,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir dryproj printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push dryproj >/dev/null 2>&1 + m_make_v1_only dryproj run "$SECRETS_BIN" migrate --dry-run [ "$status" -eq 0 ] [[ "$output" == *"would migrate"* ]] || false @@ -156,6 +157,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir cfproj printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push cfproj >/dev/null 2>&1 + m_make_v1_only cfproj local old; old=$(ls "$SECRETS_DIR/cfproj/external/"*.gradle-properties.age) run "$SECRETS_BIN" migrate [ "$status" -eq 0 ] @@ -171,6 +173,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir idemproj printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push idemproj >/dev/null 2>&1 + m_make_v1_only idemproj "$SECRETS_BIN" migrate >/dev/null 2>&1 run "$SECRETS_BIN" migrate [ "$status" -eq 0 ] @@ -187,6 +190,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir nomanifestblob printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push nomanifestblob >/dev/null 2>&1 + m_make_v1_only nomanifestblob rm -f .secrets.json # simulate a pre-manifest project run "$SECRETS_BIN" migrate [ "$status" -eq 0 ] @@ -202,6 +206,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir staleblob printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push staleblob >/dev/null 2>&1 + m_make_v1_only staleblob # The blob is now in the store. Drop the external from the project's manifest # entirely (and remove the legacy file) so NO manifest declares it. printf '{"version":2,"dotenv":[".env",".env.staging"]}\n' > .secrets.json @@ -262,6 +267,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir failverify printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push failverify >/dev/null 2>&1 + m_make_v1_only failverify "$SECRETS_BIN" migrate >/dev/null 2>&1 # corrupt the v2 twin so verify --all fails printf 'garbage' > "$SECRETS_DIR/failverify/external/"*.properties.age @@ -280,6 +286,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir untwinned printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push untwinned >/dev/null 2>&1 + m_make_v1_only untwinned # do NOT migrate — leave the v1 blob with no twin run "$SECRETS_BIN" migrate --finalize --yes [ "$status" -eq 1 ] @@ -310,6 +317,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir tagproj printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push tagproj >/dev/null 2>&1 + m_make_v1_only tagproj "$SECRETS_BIN" migrate >/dev/null 2>&1 "$SECRETS_BIN" migrate --finalize --yes >/dev/null 2>&1 local tag; tag=$(git -C "$SECRETS_DIR" tag | grep '^pre-v2-migrate-') @@ -325,6 +333,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir confproj printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push confproj >/dev/null 2>&1 + m_make_v1_only confproj "$SECRETS_BIN" migrate >/dev/null 2>&1 run bash -c "echo '' | $SECRETS_BIN migrate --finalize" [ "$status" -eq 1 ] @@ -408,6 +417,7 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > create_project_dir needsmig printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files "$SECRETS_BIN" push needsmig >/dev/null 2>&1 # v1 blob, no twin yet + m_make_v1_only needsmig run "$SECRETS_BIN" migrate --status [ "$status" -ne 0 ] # not finalize-ready [[ "$output" == *"needsmig"* ]] || false From 040782cad09763d2e93d4a09fbe37f4f91431e8e Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 10:44:34 -0700 Subject: [PATCH 38/70] feat: secrets migrate --status surfaces v2-only externals (coverage, EGB-712) --- secrets | 15 ++++++++++++--- test/migrate.bats | 11 +++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/secrets b/secrets index ed6e1f4..68e7112 100755 --- a/secrets +++ b/secrets @@ -2242,12 +2242,21 @@ _migrate_status() { new="${f%.gradle-properties.age}.properties.age" [ -f "$new" ] || untwinned=$((untwinned + 1)) done < <(find "$dir" -type f -name '*.gradle-properties.age' 2>/dev/null) + # v2-only externals: a .properties.age with no .gradle-properties.age twin — + # old (v1) clients cannot read these (the additive-v2 forcing function). + local v2only=0 pf + while IFS= read -r pf; do + [ -f "$pf" ] || continue + [ -f "${pf%.properties.age}.gradle-properties.age" ] || v2only=$((v2only + 1)) + done < <(find "$dir" -type f -name '*.properties.age' 2>/dev/null) + local v2note="" + [ "$v2only" -gt 0 ] && v2note=" [$v2only v2-only — old clients not served]" if [ "$v1" -eq 0 ]; then - echo " $project: v2-ready (no v1 properties blobs)" + echo " $project: v2-ready (no v1 properties blobs)$v2note" elif [ "$untwinned" -eq 0 ]; then - echo " $project: migrated ($v1 v1 blob(s), all twinned)" + echo " $project: migrated ($v1 v1 blob(s), all twinned)$v2note" else - echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'" + echo " $project: NEEDS MIGRATE ($untwinned of $v1 v1 blob(s) un-twinned) — cd into the project and run 'secrets migrate'$v2note" any_untwinned=1 fi done diff --git a/test/migrate.bats b/test/migrate.bats index 281a281..4ef2084 100644 --- a/test/migrate.bats +++ b/test/migrate.bats @@ -437,6 +437,17 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [[ "$output" == *"Finalize-ready"* ]] || false } +@test "migrate --status counts v2-only externals (old clients not served)" { + make_v1_store + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir v2onlyext + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push v2onlyext >/dev/null 2>&1 # v2-only (fresh, no v1 twin) + run "$SECRETS_BIN" migrate --status + [ "$status" -eq 0 ] # no v1 blobs -> finalize-ready + [[ "$output" == *"v2-only"* ]] || false # surfaced as v2-only coverage +} + @test "migrate --status on an already-v2 store says nothing to do" { init_with_remote create_project_dir v2status From 9b9af2f30c54468eceb98ef1580a9dbdf6c24d9b Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 10:52:34 -0700 Subject: [PATCH 39/70] docs: additive-v2 propagation + optional-GC finalize; bump 0.7.0.0 (EGB-712) --- CHANGELOG.md | 23 +++++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 15 ++++++++++++++- VERSION | 2 +- secrets | 2 +- 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 774c55a..ee8c3b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.0.0] - 2026-06-08 + +### Changed + +- **Additive store-format v2 (EGB-712)** — upgraded `secrets` clients now read + either external blob suffix (`.properties.age` or the legacy + `.gradle-properties.age`) and **dual-write** a `properties` external whenever a + v1 twin already exists in the store. Existing externals keep working for + teammates on an older `secrets`; only a brand-new `properties` external is + written v2-only (a gentle "upgrade to see it" forcing function). dotenv and + whole-`file` externals are unchanged across formats and always propagate. +- **`secrets migrate --finalize` is now optional GC**, not a required milestone. + Because clients dual-write and read-fall-back, no teammate is ever cut off by + *not* finalizing; finalize only reclaims the duplicate v1 blobs, and stays + deferrable indefinitely. Its safety gates are unchanged. This defuses the + cross-machine "all clients must be v2 before finalize" coordination gate. + +### Added + +- **`secrets migrate --status`** now reports `v2-only` externals per project + (the ones an un-upgraded client cannot read), so you can see the forcing + function's footprint at a glance. + ## [0.6.1.0] - 2026-06-08 ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 176fd12..1a8dba4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. -- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker; `_external_blob_suffix(type)` is the single source of truth for the suffix (push/pull/verify all route through it, so v1 and v2 stores never disagree on where a blob lives). `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710). The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker. **Additive v2 (EGB-712):** reads resolve a `properties` blob by trying `.properties.age` then falling back to `.gradle-properties.age` (`_resolve_external_blob_read`); writes dual-write a `properties` external only when a v1 twin already exists in the store (`_external_blob_write_targets`), so existing externals keep old clients fresh while brand-new externals are written v2-only (a gentle forcing function). Blob location no longer depends on the marker — the old `_external_blob_suffix` is gone. `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE, plus a `v2-only` count of externals old clients can't read), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710/EGB-712). **Under additive v2 (EGB-712) `--finalize` is now OPTIONAL GC, not a required milestone:** because upgraded clients dual-write existing externals and read-fall-back, *not* finalizing never cuts anyone off — finalize only reclaims the duplicate v1 blobs and stays deferrable indefinitely (defusing the cross-machine coordination gate). dotenv and `file` blobs are identical across formats, so they always propagate to old clients; only a brand-new `properties` external is v2-only. The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` diff --git a/README.md b/README.md index ee177b3..61a432d 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,20 @@ secrets clear | `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | | `secrets migrate [--dry-run]` | Copy-forward this project's encrypted blobs to store format v2 (non-destructive; manifest-free; `--dry-run` previews) | | `secrets migrate --status` | Survey every project's v2 readiness; exits non-zero until the whole store is finalize-ready | -| `secrets migrate --finalize` | Drop the old v1 blobs and mark the store v2 — runs once, store-wide, after `verify` is green and every machine is upgraded | +| `secrets migrate --finalize` | **Optional GC** — drop the old v1 blobs and mark the store pure v2. Never required: upgraded clients dual-write and read-fall-back, so not finalizing never cuts anyone off | + +### Upgrading: do teammates on an older `secrets` get new secrets? + +Store-format v2 is **additive** — an upgraded client reads either blob suffix and keeps the old (v1) suffix alive for externals that already existed, so you almost never have to coordinate an upgrade: + +| Secret type | Old client gets it? | +|---|---| +| `.env` / `.env.*` / `.dev.vars` | **Yes, always** (blob name is identical across formats) | +| whole-file external (`file`) | **Yes, always** | +| `properties` external that already existed | **Yes** (dual-written so old clients stay fresh) | +| brand-new `properties` external | **No — must upgrade `secrets`** (the gentle forcing function) | + +"Upgrade your secrets" = `git pull` the tool clone (binary ≥ 0.6.0.0) and/or `secrets migrate` the store. A read-only teammate only needs the tool `git pull`. ### Automatic project detection diff --git a/VERSION b/VERSION index 44e7f9a..7b86566 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1.0 +0.7.0.0 diff --git a/secrets b/secrets index 68e7112..52b6129 100755 --- a/secrets +++ b/secrets @@ -2389,7 +2389,7 @@ Usage: secrets verify --all Decrypt-test every blob in every project (integrity gate) secrets migrate [--dry-run] Copy-forward this project's v1 blobs to store format v2 secrets migrate --status Survey every project's v2 readiness (finalize gate) - secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify) + secrets migrate --finalize Optional GC: drop v1 blobs and mark the store pure v2 secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` From 367b70cba1bc49c981c1fc481901abd946f76e30 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 11:56:47 -0700 Subject: [PATCH 40/70] docs: EGB-713 version-skew nudge plan --- .../2026-06-08-egb-713-version-skew-nudge.md | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-08-egb-713-version-skew-nudge.md diff --git a/docs/superpowers/plans/2026-06-08-egb-713-version-skew-nudge.md b/docs/superpowers/plans/2026-06-08-egb-713-version-skew-nudge.md new file mode 100644 index 0000000..61d1ce5 --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-egb-713-version-skew-nudge.md @@ -0,0 +1,300 @@ +# EGB-713: Version-skew nudge Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:executing-plans / subagent-driven-development. Steps use `- [ ]`. + +**Goal:** Warn (non-fatally) when the active store was last written by a newer `secrets` version than the running client, so a behind user is told to update — the loud counterpart to EGB-712's quiet forcing function. + +**Architecture:** Stamp the store with the highest writer `VERSION` seen (`$SECRETS_DIR/.secrets-writer-version`, committed, monotonic) on every store-committing write. On any store command, compare that stamp to the client's own `VERSION` (read from `$SCRIPT_DIR/VERSION`); if the stamp is newer, print a one-time stderr nudge. Legacy stores with no stamp are silent. + +**Tech Stack:** bash 3.2 (`secrets`); bats. Spec/idea: ticket **EGB-713**. + +## Background (verified against `main`, post-EGB-712) +- `SCRIPT_DIR` already defined at `secrets:21` (`$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)`). The repo-root `VERSION` file lives next to the script. +- Store-committing write sites (each does `git -C "$SECRETS_DIR" add -A`): `commit_and_push_secrets()` `secrets:1265` (push), rekey `secrets:1822`, migrate copy-forward `secrets:2215`, finalize no-v1 `secrets:2305`, finalize drop `secrets:2340`. Plus `cmd_init` (born store) and `cmd_rm`. +- `ensure_store_protections()` (`secrets:1144`) is shared with pull (read) — do NOT stamp there. +- `check_initialized()` (`secrets:54`) early-returns when `$SECRETS_DIR/.git` exists and is called by push/pull/list/rm/rekey/verify/migrate — the natural warning hook. +- `cmd_which()` (`secrets:1945`) prints `format: v$(_store_format)` — add the writer-version line here. +- `.gitignore` only ignores `key.txt`, so `.secrets-writer-version` commits normally. +- VERSION currently `0.7.0.0` → bump to `0.7.1.0`. +- Baseline: `bats test/` = 246 passing. New tests land in a new file `test/version.bats`. + +--- + +## Task 1: Version helpers + comparator (TDD) + +**Files:** `secrets`, `test/version.bats` (new). + +- [ ] Step 1: Create `test/version.bats` testing the comparator via a tiny harness that sources the script's functions is awkward (the script runs main). Instead test through observable behavior in later tasks; for the comparator, add a hidden debug subcommand is overkill. Use this approach: test `_version_gt` indirectly by exporting it is not possible. So test the comparator by adding the helpers and a **`secrets __vercmp `** internal is overkill too. Decision: test the comparator's *effect* in Task 3 (warning) and Task 2 (stamp monotonicity), which exercise it end-to-end. For Task 1, write the helpers and verify with a one-off `bash -c` sourcing guard. + +Add to `test/version.bats`: +```bash +load test_helper + +# Exercises the comparator through a bash subshell that defines the same logic +# the script uses, guarding the numeric (not lexical) ordering contract. +@test "version comparator orders 0.7.0.0 < 0.10.0.0 numerically" { + run bash -c ' + _version_gt() { + local a="$1" b="$2" i ai bi; local -a af bf + IFS=. read -r -a af <<< "$a"; IFS=. read -r -a bf <<< "$b" + for i in 0 1 2 3; do + ai=${af[$i]:-0}; ai=${ai//[!0-9]/}; [ -n "$ai" ] || ai=0 + bi=${bf[$i]:-0}; bi=${bi//[!0-9]/}; [ -n "$bi" ] || bi=0 + if [ "$((10#$ai))" -gt "$((10#$bi))" ]; then return 0; fi + if [ "$((10#$ai))" -lt "$((10#$bi))" ]; then return 1; fi + done; return 1 + } + _version_gt 0.10.0.0 0.7.0.0 && echo "10gt7" + _version_gt 0.7.0.0 0.10.0.0 || echo "7not_gt_10" + _version_gt 0.7.0.0 0.7.0.0 || echo "equal_not_gt" + ' + [ "$status" -eq 0 ] + [[ "$output" == *"10gt7"* ]] || false + [[ "$output" == *"7not_gt_10"* ]] || false + [[ "$output" == *"equal_not_gt"* ]] || false +} +``` + +- [ ] Step 2: Run `bats test/version.bats` → PASS (pins the contract the script must match). + +- [ ] Step 3: Add the helpers to `secrets` (near `_store_format`, after `SCRIPT_DIR`/version constants — place after the `MANIFEST_VERSION=2` area or near `_store_format`): +```bash +# The running client's own version, read from the VERSION file shipped beside +# the script. Empty/"0.0.0.0" if absent (e.g. an odd install) — treated as +# "unknown/oldest" so a missing VERSION never triggers a spurious nudge. +_client_version() { + local v="" + [ -f "$SCRIPT_DIR/VERSION" ] && v=$(head -1 "$SCRIPT_DIR/VERSION" 2>/dev/null | tr -d '\r\n[:space:]') + printf '%s' "${v:-0.0.0.0}" +} + +# Numeric four-field (MAJOR.MINOR.PATCH.MICRO) compare. Returns 0 iff $1 > $2. +# Per-field numeric (so 0.10.0.0 > 0.7.0.0); missing/garbage fields → 0. +_version_gt() { + local a="$1" b="$2" i ai bi; local -a af bf + IFS=. read -r -a af <<< "$a"; IFS=. read -r -a bf <<< "$b" + for i in 0 1 2 3; do + ai=${af[$i]:-0}; ai=${ai//[!0-9]/}; [ -n "$ai" ] || ai=0 + bi=${bf[$i]:-0}; bi=${bi//[!0-9]/}; [ -n "$bi" ] || bi=0 + if [ "$((10#$ai))" -gt "$((10#$bi))" ]; then return 0; fi + if [ "$((10#$ai))" -lt "$((10#$bi))" ]; then return 1; fi + done + return 1 +} + +WRITER_VERSION_FILE_NAME=".secrets-writer-version" +# Highest client version recorded as having written to the store (empty if the +# store predates this feature — "minus the initial builds", silent by design). +_store_writer_version() { + local f="$SECRETS_DIR/$WRITER_VERSION_FILE_NAME" + [ -f "$f" ] && head -1 "$f" 2>/dev/null | tr -d '\r\n[:space:]' +} +``` + +- [ ] Step 4: `bash -n secrets` parses; `bats test/` still 246 + 1 (the comparator test) = 247. +- [ ] Step 5: Commit: `git add secrets test/version.bats && git commit -m "feat: version helpers + numeric comparator (EGB-713)"` + +--- + +## Task 2: Stamp the writer-version on write (TDD) + +**Files:** `secrets`, `test/version.bats`. + +- [ ] Step 1: Add tests: +```bash +@test "push stamps the store writer-version with the client version" { + init_with_remote + create_project_dir wvstamp + "$SECRETS_BIN" push wvstamp >/dev/null 2>&1 + [ -f "$SECRETS_DIR/.secrets-writer-version" ] + run cat "$SECRETS_DIR/.secrets-writer-version" + [ "$output" = "$(cat "$(dirname "$SECRETS_BIN")/VERSION")" ] +} + +@test "writer-version stamp is monotonic (a push never lowers a higher stamp)" { + init_with_remote + create_project_dir wvmono + printf '9.9.9.9\n' > "$SECRETS_DIR/.secrets-writer-version" + "$SECRETS_BIN" push wvmono >/dev/null 2>&1 + run cat "$SECRETS_DIR/.secrets-writer-version" + [ "$output" = "9.9.9.9" ] # not lowered to the client's version +} + +@test "writer-version stamp is committed, not gitignored" { + init_with_remote + create_project_dir wvcommit + "$SECRETS_BIN" push wvcommit >/dev/null 2>&1 + run bash -c "git -C $SECRETS_DIR ls-files | grep -qx .secrets-writer-version" + [ "$status" -eq 0 ] +} +``` + +- [ ] Step 2: Run `bats test/version.bats -f "stamp"` → FAIL (no stamping yet). + +- [ ] Step 3: Add the stamp helper (after `_store_writer_version`): +```bash +# Raise the store's recorded writer-version to the client's version (monotonic; +# never lowers it). Called right before each store-committing `git add -A` so +# the stamp rides the same commit. Read paths (pull) never call this. +_stamp_writer_version() { + local cur cli + cur=$(_store_writer_version) + cli=$(_client_version) + if [ -z "$cur" ] || _version_gt "$cli" "$cur"; then + printf '%s\n' "$cli" > "$SECRETS_DIR/$WRITER_VERSION_FILE_NAME" + fi +} +``` + +- [ ] Step 4: Call `_stamp_writer_version` immediately before each store-committing `git -C "$SECRETS_DIR" add -A`: + - `secrets:1265` (in `commit_and_push_secrets`, before `git add -A`) + - `secrets:1822` (rekey) + - `secrets:2215` (migrate copy-forward) + - `secrets:2305` (finalize, no-v1 path) + - `secrets:2340` (finalize, drop path) + Also in `cmd_init`, after the store repo is created and before its first commit (so a born store records its version), and in `cmd_rm` before its commit. + Each insertion is the single line ` _stamp_writer_version` at the matching indentation directly above the `git ... add -A` (or before the `git ... commit` where there's no add -A, e.g. rm/init — there, stamp then ensure it's staged via the existing add/commit). + +- [ ] Step 5: `bats test/version.bats` → all pass. `bats test/` → 250 (247 + 3). +- [ ] Step 6: Commit: `git add secrets test/version.bats && git commit -m "feat: stamp store writer-version on write, monotonic (EGB-713)"` + +--- + +## Task 3: Skew warning on command (TDD) + +**Files:** `secrets`, `test/version.bats`. + +- [ ] Step 1: Add tests: +```bash +@test "a store written by a newer version warns on a command (non-fatal)" { + init_with_remote + create_project_dir skewwarn + "$SECRETS_BIN" push skewwarn >/dev/null 2>&1 + printf '99.0.0.0\n' > "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] # non-fatal + [[ "$output" == *"newer"* || "$output" == *"update"* ]] || false +} + +@test "a store at the same/older version is silent" { + init_with_remote + create_project_dir noskew + "$SECRETS_BIN" push noskew >/dev/null 2>&1 # stamp == client version + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" != *"update your secrets"* ]] || false +} + +@test "a store with no writer-version marker is silent (legacy store)" { + init_with_remote + create_project_dir legacynostamp + "$SECRETS_BIN" push legacynostamp >/dev/null 2>&1 + rm -f "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" != *"update your secrets"* ]] || false +} +``` + +- [ ] Step 2: Run `bats test/version.bats -f "skew\|silent\|legacy"` → the "newer" test FAILS (no warning yet). + +- [ ] Step 3: Add the skew check (after `_stamp_writer_version`): +```bash +# Warn ONCE per invocation if the store was last written by a newer client than +# us. Non-fatal (read/write paths keep their exit codes). Silent when the store +# carries no writer-version (legacy) or is same/older than us. +_VERSION_SKEW_WARNED=0 +_check_store_version_skew() { + [ "$_VERSION_SKEW_WARNED" = 1 ] && return 0 + local sv cv + sv=$(_store_writer_version) + [ -n "$sv" ] || return 0 + cv=$(_client_version) + if _version_gt "$sv" "$cv"; then + _VERSION_SKEW_WARNED=1 + echo "NOTE: this store was last written by secrets v$sv; you're on v$cv." >&2 + echo " Update your secrets tool: git -C \"$SCRIPT_DIR\" pull" >&2 + fi + return 0 +} +``` + +- [ ] Step 4: Hook it into `check_initialized` — change `secrets:55-57`: +```bash + if [ -d "$SECRETS_DIR/.git" ]; then + return + fi +``` +to: +```bash + if [ -d "$SECRETS_DIR/.git" ]; then + _check_store_version_skew + return + fi +``` + +- [ ] Step 5: `bats test/version.bats` → all pass. `bats test/` → 253. +- [ ] Step 6: Commit: `git add secrets test/version.bats && git commit -m "feat: warn on store version skew (once per invocation, EGB-713)"` + +--- + +## Task 4: `secrets which` surfaces the writer-version (TDD) + +**Files:** `secrets`, `test/version.bats`. + +- [ ] Step 1: Add test: +```bash +@test "which prints the store writer-version and a behind note" { + init_with_remote + create_project_dir whichwv + "$SECRETS_BIN" push whichwv >/dev/null 2>&1 + printf '99.0.0.0\n' > "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"written-by: v99.0.0.0"* ]] || false + [[ "$output" == *"behind"* || "$output" == *"update"* ]] || false +} +``` + +- [ ] Step 2: Run → FAIL (which doesn't print written-by). + +- [ ] Step 3: In `cmd_which`, after the `echo "format: v$(_store_format)"` line (`secrets:1951`), add: +```bash + local _wv; _wv=$(_store_writer_version) + if [ -n "$_wv" ]; then + local _cv; _cv=$(_client_version) + if _version_gt "$_wv" "$_cv"; then + echo "written-by: v$_wv (you're on v$_cv — behind; run: git -C \"$SCRIPT_DIR\" pull)" + else + echo "written-by: v$_wv" + fi + fi +``` +Note: `cmd_which` calls `resolve_store` but may not call `check_initialized`, so this also avoids double-printing the skew NOTE; the `which` line is the dedicated surface. + +- [ ] Step 4: `bats test/version.bats` → pass. `bats test/` → 254. +- [ ] Step 5: Commit: `git add secrets test/version.bats && git commit -m "feat: secrets which shows store writer-version + behind note (EGB-713)"` + +--- + +## Task 5: Docs + version bump + +**Files:** `secrets` (cmd_help unchanged unless adding a note), `CLAUDE.md`, `README.md`, `VERSION`, `CHANGELOG.md`. + +- [ ] Step 1: `CLAUDE.md` — add to the "Store format" bullet a sentence on the writer-version: a committed `.secrets-writer-version` records the highest client `VERSION` that has written (monotonic, stamped on store-committing writes); commands warn once (stderr, non-fatal) when the store's stamp exceeds the running client, and `secrets which` shows `written-by: vN`. Legacy stores (no marker) are silent. (EGB-713.) +- [ ] Step 2: `README.md` — under the upgrading section, note that an out-of-date `secrets` prints a one-line "update" nudge when it touches a store newer than itself. +- [ ] Step 3: `VERSION` → `0.7.1.0`. +- [ ] Step 4: `CHANGELOG.md` — new `## [0.7.1.0] - 2026-06-08` with an Added entry for the version-skew nudge + `secrets which` writer-version line. +- [ ] Step 5: `bats test/` → all green (254). `./secrets which` against a scratch store renders (covered by tests). +- [ ] Step 6: Commit: `git add secrets CLAUDE.md README.md VERSION CHANGELOG.md && git commit -m "docs: version-skew nudge + writer-version; bump 0.7.1.0 (EGB-713)"` + +--- + +## Self-review vs ticket AC +- "Newer stamp → nudge; same/older → silent" → Task 3. +- "No marker → silent (legacy)" → Task 3 + `_store_writer_version` empty. +- "Monotonic, committed" → Task 2. +- "Numeric comparator (0.7.0.0 < 0.10.0.0)" → Task 1. +- "Non-fatal, never changes read/pull exit codes" → Task 3 (`_check_store_version_skew` always `return 0`). +- "`which` surfaces it" → Task 4. "Warn once per invocation" → `_VERSION_SKEW_WARNED` guard. +- bash 3.2: `read -a`, `local -a`, `10#`, parameter strips — all 3.2-safe. From 5461418c5d060dce20acc68de7cdc54bf28e2494 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 12:07:50 -0700 Subject: [PATCH 41/70] =?UTF-8?q?feat:=20version-skew=20nudge=20=E2=80=94?= =?UTF-8?q?=20stamp=20store=20writer-version,=20warn=20when=20behind=20(EG?= =?UTF-8?q?B-713)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- secrets | 81 +++++++++++++++++++++++++++++++++++ test/version.bats | 106 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 test/version.bats diff --git a/secrets b/secrets index 52b6129..b2f1559 100755 --- a/secrets +++ b/secrets @@ -53,6 +53,7 @@ check_cmd() { check_initialized() { if [ -d "$SECRETS_DIR/.git" ]; then + _check_store_version_skew return fi if [ "$STORE_SOURCE" != "default" ]; then @@ -539,6 +540,72 @@ _store_format() { echo 1 } +# ─── Client/store version skew (EGB-713) ────────────────────────────── +# +# The running client's own version, read from the VERSION file shipped beside +# the script. Empty/"0.0.0.0" if absent — treated as "unknown/oldest" so a +# missing VERSION never triggers a spurious nudge. +_client_version() { + local v="" + [ -f "$SCRIPT_DIR/VERSION" ] && v=$(head -1 "$SCRIPT_DIR/VERSION" 2>/dev/null | tr -d '\r\n[:space:]') + printf '%s' "${v:-0.0.0.0}" +} + +# Numeric four-field (MAJOR.MINOR.PATCH.MICRO) compare. Returns 0 iff $1 > $2. +# Per-field numeric (so 0.10.0.0 > 0.7.0.0); missing/garbage fields → 0. +_version_gt() { + local a="$1" b="$2" i ai bi; local -a af bf + IFS=. read -r -a af <<< "$a"; IFS=. read -r -a bf <<< "$b" + for i in 0 1 2 3; do + ai=${af[$i]:-0}; ai=${ai//[!0-9]/}; [ -n "$ai" ] || ai=0 + bi=${bf[$i]:-0}; bi=${bi//[!0-9]/}; [ -n "$bi" ] || bi=0 + if [ "$((10#$ai))" -gt "$((10#$bi))" ]; then return 0; fi + if [ "$((10#$ai))" -lt "$((10#$bi))" ]; then return 1; fi + done + return 1 +} + +WRITER_VERSION_FILE_NAME=".secrets-writer-version" +# Highest client version recorded as having written to the store (empty if the +# store predates this feature — "minus the initial builds", silent by design). +_store_writer_version() { + local f="$SECRETS_DIR/$WRITER_VERSION_FILE_NAME" + if [ -f "$f" ]; then + head -1 "$f" 2>/dev/null | tr -d '\r\n[:space:]' + fi + return 0 +} + +# Raise the store's recorded writer-version to the client's version (monotonic; +# never lowers it). Called right before each store-committing `git add -A` so +# the stamp rides the same commit. Read paths (pull) never call this. +_stamp_writer_version() { + local cur cli + cur=$(_store_writer_version) + cli=$(_client_version) + if [ -z "$cur" ] || _version_gt "$cli" "$cur"; then + printf '%s\n' "$cli" > "$SECRETS_DIR/$WRITER_VERSION_FILE_NAME" + fi +} + +# Warn ONCE per invocation if the store was last written by a newer client than +# us. Non-fatal (read/write paths keep their exit codes). Silent when the store +# carries no writer-version (legacy) or is same/older than us. +_VERSION_SKEW_WARNED=0 +_check_store_version_skew() { + [ "$_VERSION_SKEW_WARNED" = 1 ] && return 0 + local sv cv + sv=$(_store_writer_version) + [ -n "$sv" ] || return 0 + cv=$(_client_version) + if _version_gt "$sv" "$cv"; then + _VERSION_SKEW_WARNED=1 + echo "NOTE: this store was last written by secrets v$sv; you're on v$cv." >&2 + echo " Update your secrets tool: git -C \"$SCRIPT_DIR\" pull" >&2 + fi + return 0 +} + # Resolve the on-disk path of an external blob for READING. Tries the v2 suffix # (.properties.age) first, then falls back to the v1 (.gradle-properties.age) for # `properties` externals, so an upgraded client finds the blob whichever format @@ -1262,6 +1329,7 @@ commit_and_push_secrets() { # store missing the key.txt line would stage and push the private key. ensure_store_protections + _stamp_writer_version git -C "$SECRETS_DIR" add -A if git -C "$SECRETS_DIR" diff --cached --quiet 2>/dev/null; then info "No changes to push (secrets unchanged)" @@ -1819,6 +1887,7 @@ cmd_rekey() { # Commit and push (heal .gitignore first so add -A can't stage key.txt) ensure_store_protections + _stamp_writer_version git -C "$SECRETS_DIR" add -A git -C "$SECRETS_DIR" commit -m "rekey all secrets" >/dev/null if git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1; then @@ -1949,6 +2018,15 @@ cmd_which() { # EGB-700 (folded into EGB-703): surface the store format so users can tell # v1 from v2 during the migration window. v1 = legacy store, no format marker. echo "format: v$(_store_format)" + local _wv; _wv=$(_store_writer_version) + if [ -n "$_wv" ]; then + local _cv; _cv=$(_client_version) + if _version_gt "$_wv" "$_cv"; then + echo "written-by: v$_wv (you're on v$_cv — behind; run: git -C \"$SCRIPT_DIR\" pull)" + else + echo "written-by: v$_wv" + fi + fi # v2 manifest (.secrets.json): validate and summarize. Validation here # is deliberately fatal (symlink / malformed / unsupported version) so @@ -2212,6 +2290,7 @@ _migrate_project() { return 0 fi ensure_store_protections + _stamp_writer_version git -C "$SECRETS_DIR" add -A git -C "$SECRETS_DIR" commit -m "migrate: copy-forward v2 twins for $project" >/dev/null 2>&1 || true # Push the twins so a --finalize on another machine sees them (finalize @@ -2302,6 +2381,7 @@ $untwinned cd into each project and run 'secrets migrate', then re-run 'secrets # No v1 blobs at all — just stamp the marker (dotenv/file-only store). printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" ensure_store_protections + _stamp_writer_version git -C "$SECRETS_DIR" add -A git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2" >/dev/null 2>&1 || true git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true @@ -2337,6 +2417,7 @@ $untwinned cd into each project and run 'secrets migrate', then re-run 'secrets done < <(find "$SECRETS_DIR" -type f -name '*.gradle-properties.age') ensure_store_protections + _stamp_writer_version git -C "$SECRETS_DIR" add -A git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2 (drop $v1count v1 blob(s))" >/dev/null 2>&1 || true git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true diff --git a/test/version.bats b/test/version.bats new file mode 100644 index 0000000..fa12630 --- /dev/null +++ b/test/version.bats @@ -0,0 +1,106 @@ +#!/usr/bin/env bats +# EGB-713 version-skew nudge: writer-version stamp, numeric comparator, skew +# warning, `which` surface. bash 3.2: every standalone [[ ]] ends with || false. + +load test_helper + +VERSION_FILE() { echo "$(cd "$(dirname "$SECRETS_BIN")" && pwd)/VERSION"; } + +# ─── comparator contract ────────────────────────────────────────────── + +@test "version comparator orders 0.7.0.0 < 0.10.0.0 numerically (not lexically)" { + run bash -c ' + _version_gt() { + local a="$1" b="$2" i ai bi; local -a af bf + IFS=. read -r -a af <<< "$a"; IFS=. read -r -a bf <<< "$b" + for i in 0 1 2 3; do + ai=${af[$i]:-0}; ai=${ai//[!0-9]/}; [ -n "$ai" ] || ai=0 + bi=${bf[$i]:-0}; bi=${bi//[!0-9]/}; [ -n "$bi" ] || bi=0 + if [ "$((10#$ai))" -gt "$((10#$bi))" ]; then return 0; fi + if [ "$((10#$ai))" -lt "$((10#$bi))" ]; then return 1; fi + done; return 1 + } + _version_gt 0.10.0.0 0.7.0.0 && echo "10gt7" + _version_gt 0.7.0.0 0.10.0.0 || echo "7not_gt_10" + _version_gt 0.7.0.0 0.7.0.0 || echo "equal_not_gt" + _version_gt 1.0.0.0 0.9.9.9 && echo "major_wins" + ' + [ "$status" -eq 0 ] + [[ "$output" == *"10gt7"* ]] || false + [[ "$output" == *"7not_gt_10"* ]] || false + [[ "$output" == *"equal_not_gt"* ]] || false + [[ "$output" == *"major_wins"* ]] || false +} + +# ─── stamp on write ─────────────────────────────────────────────────── + +@test "push stamps the store writer-version with the client version" { + init_with_remote + create_project_dir wvstamp + "$SECRETS_BIN" push wvstamp >/dev/null 2>&1 + [ -f "$SECRETS_DIR/.secrets-writer-version" ] + run cat "$SECRETS_DIR/.secrets-writer-version" + [ "$output" = "$(cat "$(VERSION_FILE)")" ] +} + +@test "writer-version stamp is monotonic (a push never lowers a higher stamp)" { + init_with_remote + create_project_dir wvmono + printf '9.9.9.9\n' > "$SECRETS_DIR/.secrets-writer-version" + "$SECRETS_BIN" push wvmono >/dev/null 2>&1 + run cat "$SECRETS_DIR/.secrets-writer-version" + [ "$output" = "9.9.9.9" ] +} + +@test "writer-version stamp is committed, not gitignored" { + init_with_remote + create_project_dir wvcommit + "$SECRETS_BIN" push wvcommit >/dev/null 2>&1 + run bash -c "git -C $SECRETS_DIR ls-files | grep -qx .secrets-writer-version" + [ "$status" -eq 0 ] +} + +# ─── skew warning on command ────────────────────────────────────────── + +@test "a store written by a newer version warns on a command (non-fatal)" { + init_with_remote + create_project_dir skewwarn + "$SECRETS_BIN" push skewwarn >/dev/null 2>&1 + printf '99.0.0.0\n' > "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" == *"last written by secrets v99.0.0.0"* ]] || false + [[ "$output" == *"Update your secrets tool"* ]] || false +} + +@test "a store at the same/older version is silent" { + init_with_remote + create_project_dir noskew + "$SECRETS_BIN" push noskew >/dev/null 2>&1 + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" != *"Update your secrets tool"* ]] || false +} + +@test "a store with no writer-version marker is silent (legacy store)" { + init_with_remote + create_project_dir legacynostamp + "$SECRETS_BIN" push legacynostamp >/dev/null 2>&1 + rm -f "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" != *"Update your secrets tool"* ]] || false +} + +# ─── which surface ──────────────────────────────────────────────────── + +@test "which prints the store writer-version and a behind note" { + init_with_remote + create_project_dir whichwv + "$SECRETS_BIN" push whichwv >/dev/null 2>&1 + printf '99.0.0.0\n' > "$SECRETS_DIR/.secrets-writer-version" + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"written-by: v99.0.0.0"* ]] || false + [[ "$output" == *"behind"* ]] || false +} From 50476e19fdabeda798f6c9b0e8ecde164a8148fb Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 12:07:51 -0700 Subject: [PATCH 42/70] docs: version-skew nudge + writer-version; bump 0.7.1.0 (EGB-713) --- CHANGELOG.md | 13 +++++++++++++ CLAUDE.md | 2 +- README.md | 2 ++ VERSION | 2 +- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee8c3b6..2325a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.1.0] - 2026-06-08 + +### Added + +- **Version-skew nudge (EGB-713)** — the store now records the highest `secrets` + version that has written to it (`.secrets-writer-version`, committed, + monotonic). When you run a command against a store last written by a *newer* + `secrets` than your own, you get a one-line non-fatal stderr nudge to update + your tool; `secrets which` shows the store's `written-by:` version (and flags + when you're behind). Stores written by older builds carry no stamp and stay + silent — no false alarms. The loud counterpart to EGB-712's quiet + forcing function. + ## [0.7.0.0] - 2026-06-08 ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 1a8dba4..7a0926c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. -- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker. **Additive v2 (EGB-712):** reads resolve a `properties` blob by trying `.properties.age` then falling back to `.gradle-properties.age` (`_resolve_external_blob_read`); writes dual-write a `properties` external only when a v1 twin already exists in the store (`_external_blob_write_targets`), so existing externals keep old clients fresh while brand-new externals are written v2-only (a gentle forcing function). Blob location no longer depends on the marker — the old `_external_blob_suffix` is gone. `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE, plus a `v2-only` count of externals old clients can't read), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710/EGB-712). **Under additive v2 (EGB-712) `--finalize` is now OPTIONAL GC, not a required milestone:** because upgraded clients dual-write existing externals and read-fall-back, *not* finalizing never cuts anyone off — finalize only reclaims the duplicate v1 blobs and stays deferrable indefinitely (defusing the cross-machine coordination gate). dotenv and `file` blobs are identical across formats, so they always propagate to old clients; only a brand-new `properties` external is v2-only. The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker. **Additive v2 (EGB-712):** reads resolve a `properties` blob by trying `.properties.age` then falling back to `.gradle-properties.age` (`_resolve_external_blob_read`); writes dual-write a `properties` external only when a v1 twin already exists in the store (`_external_blob_write_targets`), so existing externals keep old clients fresh while brand-new externals are written v2-only (a gentle forcing function). Blob location no longer depends on the marker — the old `_external_blob_suffix` is gone. `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE, plus a `v2-only` count of externals old clients can't read), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710/EGB-712). **Under additive v2 (EGB-712) `--finalize` is now OPTIONAL GC, not a required milestone:** because upgraded clients dual-write existing externals and read-fall-back, *not* finalizing never cuts anyone off — finalize only reclaims the duplicate v1 blobs and stays deferrable indefinitely (defusing the cross-machine coordination gate). dotenv and `file` blobs are identical across formats, so they always propagate to old clients; only a brand-new `properties` external is v2-only. **Version-skew nudge (EGB-713):** a committed `$SECRETS_DIR/.secrets-writer-version` records the highest client `VERSION` that has written to the store (monotonic; stamped via `_stamp_writer_version` right before each store-committing `git add -A` — push/rekey/migrate/finalize — never on read paths, so it always rides a commit and never dangles to break `pull --ff-only`). `check_initialized` calls `_check_store_version_skew`, which warns once per invocation (stderr, non-fatal, `set -e`-safe) when the store's stamp is numerically greater than `_client_version` (read from `$SCRIPT_DIR/VERSION`); `secrets which` prints the `written-by:` line. Stores with no stamp (pre-EGB-713) are silent. The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` diff --git a/README.md b/README.md index 61a432d..532ba11 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,8 @@ Store-format v2 is **additive** — an upgraded client reads either blob suffix "Upgrade your secrets" = `git pull` the tool clone (binary ≥ 0.6.0.0) and/or `secrets migrate` the store. A read-only teammate only needs the tool `git pull`. +And you'll be told when you're behind: if a store was last written by a newer `secrets` than the one you're running, any command prints a one-line nudge to stderr (non-fatal) — and `secrets which` shows the store's `written-by:` version. Stores written by older builds (no version stamp) stay silent. + ### Automatic project detection When you run `secrets push` or `secrets pull` without specifying a project name, the tool figures out which project you're in by: diff --git a/VERSION b/VERSION index 7b86566..67085cc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.0.0 +0.7.1.0 From 446256caf192df5918beb0ab5df82baf2b7dce09 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 13:56:45 -0700 Subject: [PATCH 43/70] feat: secrets list --json machine-readable output (EGB-699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a --json flag to `secrets list` that emits a structured object for tooling/CI instead of the human table — feeds the EGB-671 install scripts, which need to enumerate a cloned store programmatically. Contract: {"store", "projects":[{"name","entries":[...]}]}, each entry self-describing via a type discriminator — {type:dotenv,path} or {type:external,subtype:properties|file,path}. cmd_list_json mirrors the same recursive store walk as the human list (nested /.age + external/.age); jq assembles the JSON so paths escape correctly and stdout stays pure JSON (the non-default-store hint is suppressed; jq is a hard dep only in --json mode). Tests: 7 new bats cases (dotenv, nested relpath, external properties + file subtypes, empty store, pure-stdout-under-notice, store path). Full suite 261 pass / 0 fail. VERSION 0.7.1.0 -> 0.7.2.0; CHANGELOG/README/CLAUDE.md updated. --- CHANGELOG.md | 15 +++++++++ CLAUDE.md | 4 +-- README.md | 1 + VERSION | 2 +- secrets | 73 +++++++++++++++++++++++++++++++++++++++++- test/secrets.bats | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 172 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2325a2f..4a51e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.2.0] - 2026-06-08 + +### Added + +- **`secrets list --json` (EGB-699)** — machine-readable listing for tooling and + CI. Emits a single JSON object on stdout: `{"store", "projects": [{"name", + "entries": [...]}]}`, where each entry self-describes via a `type` + discriminator — `{"type":"dotenv","path":}` or + `{"type":"external","subtype":"properties"|"file","path":}`. Reflects the + same recursive store walk as the human `list` (nested `/.age` + + `external/.age`). jq does the assembly so paths escape correctly; the + human store hint is suppressed so stdout stays pure JSON (notices → stderr). + jq is required only for `--json`. Feeds the EGB-671 install scripts, which need + to enumerate a cloned store programmatically instead of scraping the table. + ## [0.7.1.0] - 2026-06-08 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 7a0926c..608040a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ - secrets.bats # bats-core test suite (133 tests) + secrets.bats # bats-core test suite (140 tests) manifest.bats # EGB-677 .secrets.json manifest tests (78 tests) migrate.bats # EGB-703 store-format-v2 migration tests (26 tests) test_helper.bash # Shared setup/teardown @@ -107,7 +107,7 @@ The active store directory is picked by `resolve_store()` using these rules, hig Key design decisions (all driven by /autoplan review): - **Wire-in is at command scope** (`cmd_push`/`cmd_pull`), via `push_external_files` / `pull_external_files`, **not** inside `push_dir_to_project` / `pull_project_to_dir` (those loop per-workspace and `pull_project_to_dir` uses stdout as a data channel). -- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). +- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). `cmd_list --json` (EGB-699) emits the same recursive walk as a machine-readable object (`{store, projects[].entries[]}`, each entry `dotenv`→`path` or `external`→`subtype`+`path`) for tooling/CI (feeds EGB-671); jq assembles it so paths escape correctly and stdout stays pure JSON (the human store hint is suppressed; jq is a hard dep only in `--json` mode). - **Merge is pure bash, no `sed`/regex** (`merge_gradle_keys`): exact-string key comparison (avoids `beaconClerkPk` vs `beaconClerkPkTest` substring bug), value treated as opaque literal (survives `& \ /` in values). Updates a managed key in place at its first occurrence, collapses duplicates, appends new keys, preserves unrelated lines/comments/order. Continuation lines (trailing odd backslashes, tracked by `_trailing_bs_odd`) are never matched as keys. Atomic write: temp in the same dir → `chmod` to match (or `600` on create) → `mv`. Backs up to `.secrets-bak` before each merge. - **Properties separator parsing** (`_props_get`): key ends at the first `=`, `:`, or whitespace (after lstrip); handles `key=value`, `key = value`, `key:value`, `key value`; last definition wins. - **Security:** the write target comes from a committed file, so `_validate_external_target_path` locks it down — basename must be `gradle.properties`, must resolve inside `$HOME` (deepest-existing-ancestor resolved, symlink target/parent refused, `..` rejected). This blocks a malicious manifest from appending decrypted keys to `~/.gitconfig`/`~/.bashrc`. `_parse_secrets_files_manifest` rejects shell metacharacters/control chars in path and keys (path allows `[A-Za-z0-9/._~-]` only; keys allow `[A-Za-z0-9._-]` + space), mirrors the `.secrets-store` posture (no shell expansion, symlinked manifest skipped). diff --git a/README.md b/README.md index 532ba11..e345d64 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ secrets clear | `secrets clear` | Delete plaintext secret files from the current directory | | `secrets run ` | Pull secrets, run a command, then clear secrets when it exits | | `secrets list` | Show all projects that have stored secrets | +| `secrets list --json` | Same listing as a machine-readable JSON object (`{store, projects[].entries[]}`, each entry `dotenv`/`external`) for tooling and CI. JSON goes to stdout; notices to stderr | | `secrets rm ` | Delete a project's secrets from the store | | `secrets rekey` | Generate a new encryption key and re-encrypt everything | | `secrets verify [project]` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob. `[project]` overrides the store directory name; the manifest is still read from the current directory | diff --git a/VERSION b/VERSION index 67085cc..9872478 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1.0 +0.7.2.0 diff --git a/secrets b/secrets index b2f1559..3ad6c40 100755 --- a/secrets +++ b/secrets @@ -1744,9 +1744,21 @@ cmd_pull_workspaces() { } cmd_list() { + local json=0 + case "${1:-}" in + --json) json=1 ;; + "") ;; + *) die "Usage: secrets list [--json]" ;; + esac + resolve_store check_initialized + if [ "$json" -eq 1 ]; then + cmd_list_json + return + fi + local found=0 for dir in "$SECRETS_DIR"/*/; do [ -d "$dir" ] || continue @@ -1782,6 +1794,64 @@ cmd_list() { fi } +# EGB-699: machine-readable listing for tooling/CI (feeds EGB-671 install +# scripts). Contract: a single JSON object on stdout — +# {"store": "", "projects": [{"name", "entries": [...]}]} +# where each entry is {"type":"dotenv","path":} or +# {"type":"external","subtype":"properties"|"file","path":}. Mirrors the +# recursive store walk the human `list` uses (nested /.age + +# external/.age). jq does the assembly so paths are escaped correctly; +# stdout stays pure JSON (the human store hint is suppressed in this mode). +cmd_list_json() { + check_cmd jq + + { + for dir in "$SECRETS_DIR"/*/; do + [ -d "$dir" ] || continue + local project + project=$(basename "$dir") + [[ "$project" == .* ]] && continue + # Marker line so a project with zero blobs still appears (mirrors the + # human header), grouped via jq below. + printf 'project\t%s\n' "$project" + local f rel name + while IFS= read -r f; do + [ -f "$f" ] || continue + rel=${f#"$dir"} + rel=${rel%.age} + case "$rel" in + external/*) + name=${rel#external/} + case "$name" in + *.file) printf 'entry\t%s\texternal\tfile\t%s\n' "$project" "${name%.file}" ;; + *.properties) printf 'entry\t%s\texternal\tproperties\t%s\n' "$project" "${name%.properties}" ;; + *.gradle-properties) printf 'entry\t%s\texternal\tproperties\t%s\n' "$project" "${name%.gradle-properties}" ;; + *) printf 'entry\t%s\texternal\tunknown\t%s\n' "$project" "$name" ;; + esac + ;; + *) + printf 'entry\t%s\tdotenv\t\t%s\n' "$project" "$rel" + ;; + esac + done < <(find "$dir" -type f -name '*.age' | sort) + done + } | jq -R -n --arg store "$SECRETS_DIR" ' + [inputs | split("\t")] as $lines + | ($lines | map(select(.[0] == "project") | .[1]) | unique) as $names + | { + store: $store, + projects: ($names | map(. as $p | { + name: $p, + entries: [ $lines[] + | select(.[0] == "entry" and .[1] == $p) + | if .[2] == "external" + then { type: "external", subtype: .[3], path: .[4] } + else { type: "dotenv", path: .[4] } + end ] + })) + }' +} + cmd_rm() { check_cmd git resolve_store @@ -2464,6 +2534,7 @@ Usage: secrets clear -w|--workspaces Clear secrets from all workspaces in package.json secrets run [-w] Pull secrets, run command, clear secrets on exit secrets list List all projects and their secret files + secrets list --json Same listing as machine-readable JSON (for tooling/CI) secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key secrets verify [project] Check the manifest against the store + decrypt every blob @@ -2636,7 +2707,7 @@ case "${1:-help}" in cmd_run "$@" ;; add) cmd_add "${2:-}" ;; - list) cmd_list ;; + list) shift; cmd_list "$@" ;; rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; verify) shift; cmd_verify "$@" ;; diff --git a/test/secrets.bats b/test/secrets.bats index a75c0e6..3a72910 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1763,3 +1763,84 @@ file_project() { [[ "$output" == *"Extracted 1 key"* ]] || false [[ "$output" == *"Encrypted file"* ]] || false } + +# ─── EGB-699: `list --json` machine-readable output ────────────────────── + +@test "EGB-699: list --json emits valid JSON with project and dotenv entry" { + init_with_remote + create_project_dir jproj + "$SECRETS_BIN" push jproj >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + # entire stdout parses as JSON + echo "$output" | jq -e . >/dev/null + # project is present + echo "$output" | jq -e '.projects[] | select(.name == "jproj")' >/dev/null + # .env shows up as a dotenv entry + echo "$output" | jq -e '.projects[] | select(.name == "jproj") + | .entries[] | select(.type == "dotenv" and .path == ".env")' >/dev/null +} + +@test "EGB-699: list --json includes a nested dotenv relpath" { + init_with_remote + create_project_dir nestjson + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + echo "$output" | jq -e '.projects[] | select(.name == "nestjson") + | .entries[] | select(.type == "dotenv" and .path == "packages/web/.env.development")' >/dev/null +} + +@test "EGB-699: list --json marks an external properties entry with subtype" { + init_with_remote + gradle_src $'beaconClerkPkTest=pk_test_abc\n' + gradle_project gjson beaconClerkPkTest + "$SECRETS_BIN" push gjson >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + echo "$output" | jq -e '.projects[] | select(.name == "gjson") + | .entries[] | select(.type == "external" and .subtype == "properties")' >/dev/null +} + +@test "EGB-699: list --json marks an external file entry with subtype" { + init_with_remote + file_src + local dir="$WORK_DIR/fjson"; mkdir -p "$dir" + printf 'file ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + "$SECRETS_BIN" push fjson >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + echo "$output" | jq -e '.projects[] | select(.name == "fjson") + | .entries[] | select(.type == "external" and .subtype == "file")' >/dev/null +} + +@test "EGB-699: list --json on an empty store emits an empty projects array" { + "$SECRETS_BIN" init >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + echo "$output" | jq -e '.projects == []' >/dev/null +} + +@test "EGB-699: list --json keeps stdout pure JSON (notices go to stderr)" { + # The non-default-store hint normally prints to stdout in human mode; under + # --json it must not, or it would corrupt the document. Capture stdout only. + init_with_remote + create_project_dir purejson + "$SECRETS_BIN" push purejson >/dev/null 2>&1 + local json + json=$("$SECRETS_BIN" list --json 2>/dev/null) + echo "$json" | jq -e . >/dev/null +} + +@test "EGB-699: list --json reports the active store path" { + init_with_remote + create_project_dir storejson + "$SECRETS_BIN" push storejson >/dev/null 2>&1 + run "$SECRETS_BIN" list --json + [ "$status" -eq 0 ] + echo "$output" | jq -e --arg s "$SECRETS_DIR" '.store == $s' >/dev/null +} From 6319313ee40af73328b67b69f8f16df673a86367 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 16:23:50 -0700 Subject: [PATCH 44/70] feat: secrets join + init --remote + verified onboarding (EGB-671) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add second-machine onboarding as a first-class verb rather than a manual clone + key-copy sequence: - secrets join --remote --key : clone the vault, install the key at mode 600, then decrypt-test it before declaring success. An empty vault reports "nothing to verify yet" (never a false VERIFIED); a wrong key fails loudly. Reuses the audited core (resolve_store, get_pubkey, _verify_all) — no security logic re-implemented. - secrets init --remote : wire the remote and push the initial store so the upstream branch exists (fixes the commit_and_push_secrets pull --ff-only die against a brand-new empty remote). init also offers an interactive first-add of a project (default No; skipped under --yes / non-interactive). - cmd_push first-manifest scaffold writes an explicit committed options.autoAdd value, asked once when interactive (EGB-677 contract #2). - secrets pull now dies loudly when a blob fails to decrypt (all three decrypt paths) instead of warning and exiting 0 — a wrong key can't pass silently. - Interactive prompts gate on stdin AND stdout being ttys, so bats/CI never hang. - Dispatcher routes init/join args correctly; second-machine trap points at join. Tests: 20 new (join, autoAdd, pty-no-hang regression); 2 trap tests updated. --- secrets | 239 ++++++++++++++++++++++++++++++++++++++++----- test/join.bats | 145 +++++++++++++++++++++++++++ test/manifest.bats | 11 +++ test/secrets.bats | 13 +-- 4 files changed, 380 insertions(+), 28 deletions(-) create mode 100644 test/join.bats diff --git a/secrets b/secrets index 3ad6c40..219cfbf 100755 --- a/secrets +++ b/secrets @@ -1235,6 +1235,25 @@ ensure_store_protections() { cmd_init() { check_cmd age check_cmd git + + # EGB-671: flag parsing. --remote wires the encrypted-vault git remote and + # establishes an upstream branch (so the first project push won't hit the + # commit_and_push_secrets `pull --ff-only` die on a brand-new empty remote). + # --yes / non-interactive means init-only: skip the interactive first-add. + local remote="" assume_yes=false + while [ $# -gt 0 ]; do + case "$1" in + --remote) [ $# -ge 2 ] || die "--remote requires a URL" + case "$2" in --|-*) die "--remote value looks like a flag: $2" ;; esac + remote="$2"; shift 2 ;; + --remote=*) remote="${1#--remote=}" + [ -n "$remote" ] || die "--remote= requires a value"; shift ;; + --yes|-y) assume_yes=true; shift ;; + -*) die "Unknown init flag: $1. Usage: secrets init [--remote ] [--yes]" ;; + *) die "Unexpected argument to init: $1" ;; + esac + done + resolve_store if [ -d "$SECRETS_DIR/.git" ]; then @@ -1242,17 +1261,15 @@ cmd_init() { fi # Second-machine trap: a copied key.txt without a repo means the user - # should clone their existing secrets repo, not init a fresh one. - # Catch it BEFORE git init so we don't leave a half-initialized store. + # should join their existing vault, not init a fresh one. Catch it BEFORE + # git init so we don't leave a half-initialized store. if [ -f "$KEY_FILE" ]; then - # Render a runnable clone command when .secrets-store carried a remote - # URL (already sanitized by resolve_store), mirroring check_initialized. local clone_src="" [ -n "${_REMOTE_URL:-}" ] && clone_src="$_REMOTE_URL" die "Found an existing key at $KEY_FILE but no repo at $SECRETS_DIR. -If this is a second machine, don't run 'secrets init' — clone your existing secrets repo instead: +If this is a second machine, don't run 'secrets init' — join your existing vault: - git clone $clone_src $SECRETS_DIR + secrets join --remote $clone_src --key $KEY_FILE Your key file has been left untouched." fi @@ -1268,9 +1285,7 @@ Your key file has been left untouched." # Write .gitignore write_store_gitignore - # Stamp the store format (EGB-703): a fresh store is born v2 — it has no - # v1 blobs, so it is already in v2 shape. The marker is a committed, - # non-secret metadata file (NOT gitignored); the first push stages it. + # Stamp the store format (EGB-703): a fresh store is born v2. printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" # Install pre-commit hook @@ -1282,11 +1297,159 @@ Your key file has been left untouched." info "Done! Your public key is:" echo " $pubkey" + + if [ -n "$remote" ]; then + _init_wire_remote "$remote" + else + echo "" + echo "Next steps:" + echo " 1. Add a remote: secrets init --remote (or: git -C $SECRETS_DIR remote add origin )" + echo " 2. Copy $KEY_FILE to your other machine (AirDrop, scp, USB)" + echo " 3. On the other machine: secrets join --remote --key " + fi + + # Interactive first-add (EGB-671): only when truly interactive AND not --yes. + # Require BOTH stdin and stdout to be ttys — bats/CI capture a command's + # stdout (so `[ -t 1 ]` is false under automation even when stdin is still + # the terminal), which is the reliable "don't prompt" signal. Default is No. + if [ "$assume_yes" != true ] && [ -t 0 ] && [ -t 1 ] && [ -e /dev/tty ]; then + _init_first_add + fi +} + +# EGB-671: wire the encrypted-vault remote and establish an upstream branch. +# Commits the born-v2 store (so .secrets-format etc. exist on the remote) and +# push -u, so a later `secrets push` pulls --ff-only against a real upstream +# instead of dying on a non-existent branch. +_init_wire_remote() { + local remote="$1" + git -C "$SECRETS_DIR" remote add origin "$remote" + ensure_store_protections + _stamp_writer_version + git -C "$SECRETS_DIR" add -A + git -C "$SECRETS_DIR" commit -m "Initialize secrets store (format v2)" >/dev/null 2>&1 || true + local br + br=$(git -C "$SECRETS_DIR" symbolic-ref --short HEAD 2>/dev/null || echo main) + if git -C "$SECRETS_DIR" push -u origin "$br" >/dev/null 2>&1; then + info "Wired remote origin=$remote and pushed the initial store (upstream: origin/$br)." + else + info "Added remote origin=$remote, but the initial push failed." + echo " Create the PRIVATE repo first, then: git -C $SECRETS_DIR push -u origin $br" >&2 + fi echo "" - echo "Next steps:" - echo " 1. Add a remote: cd $SECRETS_DIR && git remote add origin " - echo " 2. Copy $KEY_FILE to your other machine (AirDrop, scp, USB)" - echo " 3. Run 'secrets push ' from a project directory" + echo "Next: copy $KEY_FILE to your other machine, then run there:" + echo " secrets join --remote $remote --key " +} + +# EGB-671: interactive "add your first project" post-step. Reads from /dev/tty +# so it never collides with a piped stdin. Default No. Drives the existing +# push path (cmd_push is $PWD-bound) by cd'ing into the chosen project dir. +_init_first_add() { + printf "Add a project's secrets to the vault now? [y/N] " > /dev/tty 2>/dev/null || return 0 + local ans="" + read -r ans < /dev/tty 2>/dev/null || return 0 + case "$ans" in [Yy]*) ;; *) return 0 ;; esac + printf "Path to the project directory: " > /dev/tty 2>/dev/null || return 0 + local dir="" + read -r dir < /dev/tty 2>/dev/null || return 0 + [ -n "$dir" ] || return 0 + case "$dir" in + "~") dir="$HOME" ;; + "~/"*) dir="$HOME/${dir#\~/}" ;; + esac + if [ ! -d "$dir" ]; then + echo "Not a directory: $dir — skipping first-add. Run 'secrets push' from a project later." > /dev/tty 2>/dev/null || true + return 0 + fi + ( cd "$dir" && cmd_push ) || true + return 0 +} + +# EGB-671: second-machine onboarding. Clone the vault, install the key, and +# decrypt-test it BEFORE declaring success — the one thing a hand-copied +# README sequence never did (a mis-copied key fails silently at first pull). +# Security logic (path rails on --key/--store, URL sanitization) lives in the +# audited core, reused — never re-implemented in a standalone install script. +cmd_join() { + check_cmd age + check_cmd git + + local remote="" keyfile="" + while [ $# -gt 0 ]; do + case "$1" in + --remote) [ $# -ge 2 ] || die "--remote requires a URL" + case "$2" in --|-*) die "--remote value looks like a flag: $2" ;; esac + remote="$2"; shift 2 ;; + --remote=*) remote="${1#--remote=}" + [ -n "$remote" ] || die "--remote= requires a value"; shift ;; + --key) [ $# -ge 2 ] || die "--key requires a path" + case "$2" in --|-*) die "--key value looks like a flag: $2" ;; esac + keyfile="$2"; shift 2 ;; + --key=*) keyfile="${1#--key=}" + [ -n "$keyfile" ] || die "--key= requires a value"; shift ;; + -*) die "Unknown join flag: $1. Usage: secrets join --remote --key " ;; + *) die "Unexpected argument to join: $1" ;; + esac + done + + resolve_store + + [ -n "$remote" ] || die "secrets join requires --remote + Usage: secrets join --remote --key " + [ -n "$keyfile" ] || die "secrets join requires --key + This is the age key (key.txt) from your first machine. + Usage: secrets join --remote --key " + if [ -d "$keyfile" ]; then + die "--key must point to the key FILE, not a directory: $keyfile + Did you mean: --key $keyfile/key.txt ?" + fi + [ -e "$keyfile" ] || die "Key file not found: $keyfile + Copy key.txt from your first machine (AirDrop/scp/USB) and pass its path." + [ -r "$keyfile" ] || die "Key file not readable: $keyfile" + + if [ -e "$SECRETS_DIR" ]; then + die "A store already exists at $SECRETS_DIR. + 'secrets join' clones a fresh vault — it won't clobber an existing one. + If you meant to refresh it, run 'secrets pull' instead, or remove $SECRETS_DIR first." + fi + + info "Joining vault: cloning $remote → $SECRETS_DIR" + if ! git clone "$remote" "$SECRETS_DIR" >/dev/null 2>&1; then + rm -rf "$SECRETS_DIR" + die "Failed to clone $remote + Check the URL and that you have access to the repo." + fi + + # Install the key BEFORE anything that decrypts, at mode 600. + cp "$keyfile" "$SECRETS_DIR/key.txt" + chmod 600 "$SECRETS_DIR/key.txt" + KEY_FILE="$SECRETS_DIR/key.txt" + ensure_store_protections + + if ! get_pubkey >/dev/null 2>&1; then + die "The file you passed to --key is not a valid age identity: $keyfile + Your store was cloned to $SECRETS_DIR; replace key.txt with a valid key and run 'secrets pull'." + fi + + # Verify gate: decrypt-test every blob. An EMPTY store returns 0 from + # _verify_all ("nothing to check") — that proves nothing about the key, so + # join must NOT report VERIFIED in that case (EGB-671 / E-S1b). + local blob_count + blob_count=$(find "$SECRETS_DIR" -type f -name '*.age' 2>/dev/null | wc -l | tr -d ' ') + if [ "$blob_count" -eq 0 ]; then + info "Joined $SECRETS_DIR — the vault is empty, so there's nothing to verify yet." + echo "Next: run 'secrets pull' in a project once secrets have been pushed from another machine." + return 0 + fi + if _verify_all >/dev/null 2>&1; then + info "VERIFIED — your key decrypts all $blob_count blob(s). You've joined the vault." + echo "Next: run 'secrets pull' in any project to restore its secrets." + return 0 + fi + die "Your key does NOT decrypt this vault ($blob_count blob(s) failed). + This is almost always the wrong key.txt. The store is at $SECRETS_DIR; + replace key.txt with the correct key and run 'secrets pull', or remove + $SECRETS_DIR and re-run 'secrets join' with the right --key." } # Encrypt env files from a source dir into a project path in the secrets repo. @@ -1498,8 +1661,22 @@ cmd_push() { '.dotenv = ((.dotenv // []) + $add) | .external = ((.external // []) + $ext)' "$manifest" \ | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" else - jq -n --argjson add "$add_json" --argjson ext "$absorbed_json" \ - '{version: '"$MANIFEST_VERSION"', dotenv: $add} | if ($ext | length) > 0 then .external = $ext else . end' \ + # EGB-671 / EGB-677 contract #2: scaffolding the project's FIRST manifest. + # Record an explicit, committed options.autoAdd value. Ask once when + # interactive (read from /dev/tty so a piped stdin never collides); + # otherwise write the tool default (ON) explicitly so the value is + # committed and team-shared rather than left implicit. + local autoadd_commit="true" + # Require BOTH stdin and stdout to be ttys (bats/CI capture stdout, so + # `[ -t 1 ]` is false under automation — never block a scripted push). + if [ -t 0 ] && [ -t 1 ] && [ -e /dev/tty ]; then + printf "Auto-track new env files in this project as you add them? [Y/n] " > /dev/tty 2>/dev/null || true + local _aa="" + read -r _aa < /dev/tty 2>/dev/null || _aa="" + case "$_aa" in [Nn]*) autoadd_commit="false" ;; *) autoadd_commit="true" ;; esac + fi + jq -n --argjson add "$add_json" --argjson ext "$absorbed_json" --argjson autoadd "$autoadd_commit" \ + '{version: '"$MANIFEST_VERSION"', dotenv: $add, options: {autoAdd: $autoadd}} | if ($ext | length) > 0 then .external = $ext else . end' \ | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" fi if [ "$write_adds" = true ]; then @@ -1616,9 +1793,14 @@ cmd_pull() { continue fi case "$rel" in */*) mkdir -p "$target_dir/$(dirname "$rel")" ;; esac - age -d -i "$KEY_FILE" -o "$target_dir/$rel" "$blob" + # EGB-671 (DX-3): a wrong-but-structurally-valid key must NOT fail + # silently. Die loudly on decrypt failure instead of leaving a partial. + if ! age -d -i "$KEY_FILE" -o "$target_dir/$rel" "$blob"; then + die "Failed to decrypt '$rel' with the current key ($KEY_FILE). + Wrong key for this vault? Run 'secrets verify --all' to check the key." + fi if [ ! -s "$target_dir/$rel" ]; then - echo "WARNING: Decrypted file '$rel' is empty (possibly truncated .age blob)" + echo "WARNING: Decrypted file '$rel' is empty (an empty source file, or a truncated .age blob)" fi count=$((count + 1)) done <<< "$declared" @@ -1642,10 +1824,14 @@ cmd_pull() { local name name=$(basename "$f" .age) local outfile="$target_dir/$name" - age -d -i "$KEY_FILE" -o "$outfile" "$f" + # EGB-671 (DX-3): die loudly on decrypt failure (wrong key) — never silent. + if ! age -d -i "$KEY_FILE" -o "$outfile" "$f"; then + die "Failed to decrypt '$name' with the current key ($KEY_FILE). + Wrong key for this vault? Run 'secrets verify --all' to check the key." + fi # Integrity check: verify non-empty if [ ! -s "$outfile" ]; then - echo "WARNING: Decrypted file '$name' is empty (possibly truncated .age blob)" + echo "WARNING: Decrypted file '$name' is empty (an empty source file, or a truncated .age blob)" fi count=$((count + 1)) done @@ -1674,9 +1860,13 @@ pull_project_to_dir() { local name name=$(basename "$f" .age) local outfile="$target_dir/$name" - age -d -i "$KEY_FILE" -o "$outfile" "$f" + # EGB-671 (DX-3): die loudly on decrypt failure (wrong key) — never silent. + if ! age -d -i "$KEY_FILE" -o "$outfile" "$f"; then + die "Failed to decrypt '$name' with the current key ($KEY_FILE). + Wrong key for this vault? Run 'secrets verify --all' to check the key." + fi if [ ! -s "$outfile" ]; then - echo "WARNING: Decrypted file '$name' is empty (possibly truncated .age blob)" + echo "WARNING: Decrypted file '$name' is empty (an empty source file, or a truncated .age blob)" fi count=$((count + 1)) done @@ -2523,6 +2713,10 @@ secrets — encrypted secret file sync between machines Usage: secrets init Initialize the secrets repo and generate an age key + secrets init --remote Init, wire the remote, and push the initial store + secrets join --remote --key + Join an existing vault on a new machine: clone, + install the key, and verify it decrypts the store secrets push [project] Encrypt secret files and push to the secrets repo secrets push --frozen Sync only manifest-declared files (skip auto-add) secrets push --dry-run Show what would be added/synced; change nothing @@ -2679,7 +2873,8 @@ else fi case "${1:-help}" in - init) cmd_init ;; + init) shift; cmd_init "$@" ;; + join) shift; cmd_join "$@" ;; push) if [ "${2:-}" = "-w" ] || [ "${2:-}" = "--workspaces" ]; then cmd_push_workspaces diff --git a/test/join.bats b/test/join.bats new file mode 100644 index 0000000..205570a --- /dev/null +++ b/test/join.bats @@ -0,0 +1,145 @@ +#!/usr/bin/env bats +# EGB-671: `secrets join` (second-machine onboarding) + `secrets init --remote` +# + day-2 silent-decrypt fix. Functional paths only — security-rail tests +# (path traversal on --key/--store, URL injection) are operator-local per +# .ship-policy.json and live in test/run-security.sh. + +load test_helper + +# Push a project to REMOTE_DIR and save the key, then remove the local store +# to simulate a fresh second machine. Leaves: REMOTE_DIR has blobs, +# $TEST_TMPDIR/saved-key.txt is the decrypting key, $SECRETS_DIR is gone. +_machine1_push_then_wipe() { + init_with_remote + cp "$SECRETS_DIR/key.txt" "$TEST_TMPDIR/saved-key.txt" + create_project_dir "joinproj" + "$SECRETS_BIN" push >/dev/null 2>&1 + cd "$HOME" + rm -rf "$SECRETS_DIR" +} + +# Like above but never pushes a project — remote has a store with zero blobs. +_machine1_empty_then_wipe() { + init_with_remote + cp "$SECRETS_DIR/key.txt" "$TEST_TMPDIR/saved-key.txt" + cd "$HOME" + rm -rf "$SECRETS_DIR" +} + +# ─── secrets join ──────────────────────────────────────────────────────── + +@test "join without --remote fails with usage" { + run "$SECRETS_BIN" join + [ "$status" -ne 0 ] + [[ "$output" == *"--remote"* ]] || false +} + +@test "join clones the store, installs the key at 600, verifies, and succeeds" { + _machine1_push_then_wipe + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR/saved-key.txt" + [ "$status" -eq 0 ] + [[ "$output" == *"VERIFIED"* ]] || false + [ -d "$SECRETS_DIR/.git" ] + [ -f "$SECRETS_DIR/key.txt" ] + # key installed at mode 600 + local perms + perms=$(stat -f '%Lp' "$SECRETS_DIR/key.txt" 2>/dev/null || stat -c '%a' "$SECRETS_DIR/key.txt") + [ "$perms" = "600" ] +} + +@test "join with the wrong key fails loudly and does not report VERIFIED" { + _machine1_push_then_wipe + age-keygen -o "$TEST_TMPDIR/wrong-key.txt" 2>/dev/null + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR/wrong-key.txt" + [ "$status" -ne 0 ] + [[ "$output" != *"VERIFIED"* ]] || false +} + +@test "join against an empty store reports nothing-to-verify, NOT VERIFIED" { + _machine1_empty_then_wipe + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR/saved-key.txt" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to verify"* ]] || false + [[ "$output" != *"VERIFIED"* ]] || false +} + +@test "join refuses when a store already exists at the target" { + "$SECRETS_BIN" init >/dev/null 2>&1 + cp "$SECRETS_DIR/key.txt" "$TEST_TMPDIR/saved-key.txt" + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR/saved-key.txt" + [ "$status" -ne 0 ] + [[ "$output" == *"already"* ]] || false +} + +@test "join fails clearly when the key file is missing" { + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR/nope.txt" + [ "$status" -ne 0 ] + [[ "$output" == *"key"* ]] || false +} + +@test "join detects a directory passed as --key" { + _machine1_push_then_wipe + run "$SECRETS_BIN" join --remote "$REMOTE_DIR" --key "$TEST_TMPDIR" + [ "$status" -ne 0 ] + [[ "$output" == *"key"* ]] || false +} + +# ─── secrets init --remote ──────────────────────────────────────────────── + +@test "init --remote sets origin and establishes an upstream branch" { + run "$SECRETS_BIN" init --remote "$REMOTE_DIR" + [ "$status" -eq 0 ] + run git -C "$SECRETS_DIR" remote get-url origin + [ "$status" -eq 0 ] + [ "$output" = "$REMOTE_DIR" ] + # upstream branch exists on the remote (so a later push won't ff-only die) + run git -C "$SECRETS_DIR" rev-parse --abbrev-ref '@{u}' + [ "$status" -eq 0 ] +} + +@test "init --remote then push does not die on the brand-new remote" { + "$SECRETS_BIN" init --remote "$REMOTE_DIR" >/dev/null 2>&1 + create_project_dir "freshproj" + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" != *"Fast-forward pull failed"* ]] || false +} + +@test "init with no flags still works (clean primitive)" { + run "$SECRETS_BIN" init + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/key.txt" ] +} + +@test "init does not hang on the first-add prompt when stdin is a tty but stdout is captured" { + # Regression: run-security.sh runs bats in a real terminal, so the command's + # stdin stays a tty while bats captures its stdout. The interactive first-add + # prompt must NOT fire in that shape (it gates on stdout being a tty too), + # or the whole suite hangs. Reproduce with a pty via `script`. + command -v script >/dev/null 2>&1 || skip "script (pty) not available" + # macOS/BSD syntax: `script -q `. Skip on other syntaxes. + script -q /dev/null true >/dev/null 2>&1 || skip "unsupported script syntax" + local out="$TEST_TMPDIR/pty-initout" + run timeout 10 script -q /dev/null bash -c "'$SECRETS_BIN' init > '$out' 2>&1" + [ "$status" -ne 124 ] # 124 == timeout == it hung on a prompt + run grep -c "Add a project's secrets" "$out" + [ "$output" = "0" ] +} + +# ─── day-2 silent-decrypt fix ───────────────────────────────────────────── + +@test "pull dies loudly when a blob cannot be decrypted with the current key" { + init_with_remote + create_project_dir "decryptproj" + "$SECRETS_BIN" push >/dev/null 2>&1 + # Swap in a different key so the stored blob no longer decrypts. + # (age-keygen refuses to overwrite, so generate elsewhere then copy.) + age-keygen -o "$TEST_TMPDIR/other-key.txt" 2>/dev/null + cp "$TEST_TMPDIR/other-key.txt" "$SECRETS_DIR/key.txt" + chmod 600 "$SECRETS_DIR/key.txt" + cd "$WORK_DIR/decryptproj" + rm -f .env .env.staging + run "$SECRETS_BIN" pull + [ "$status" -ne 0 ] + [[ "$output" == *"decrypt"* ]] || false +} diff --git a/test/manifest.bats b/test/manifest.bats index eb5c42a..ce16125 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -167,6 +167,17 @@ load test_helper [ "$output" = "2" ] } +@test "bootstrap: first push writes an explicit options.autoAdd value (EGB-677 contract #2)" { + init_with_remote + create_project_dir autoaddproj + # Non-interactive (bats has no tty): the prompt is skipped and the tool + # default (ON) is written explicitly so the value is committed + team-shared. + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + run jq -r '.options.autoAdd' .secrets.json + [ "$output" = "true" ] +} + @test "failed push leaves no bootstrap manifest behind" { init_with_remote mkdir -p "$WORK_DIR/emptyproj" diff --git a/test/secrets.bats b/test/secrets.bats index 3a72910..ee2928f 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1484,9 +1484,10 @@ gradle_project() { # ─── init second-machine guard + store .gitignore self-heal ──────────── -@test "init with existing key but no repo dies with clone guidance" { - # Second-machine trap: user copies key.txt into ~/.secrets, then runs - # `secrets init` instead of cloning their secrets repo. +@test "init with existing key but no repo dies with join guidance" { + # Second-machine trap (EGB-671): user copies key.txt into ~/.secrets, then + # runs `secrets init` instead of joining their existing vault. The trap now + # points at `secrets join` (the real one-command path), not a manual clone. mkdir -p "$SECRETS_DIR" age-keygen -o "$SECRETS_DIR/key.txt" 2>/dev/null # Guard against a vacuous '' = '' comparison if age-keygen failed @@ -1496,7 +1497,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone"* ]] || false + [[ "$output" == *"secrets join"* ]] || false # Must not leave a half-initialized store behind [ ! -d "$SECRETS_DIR/.git" ] # Key untouched @@ -1651,7 +1652,7 @@ gradle_project() { [[ "$output" != *"key.txt"* ]] || false } -@test "init guard renders the real clone URL when .secrets-store carries a remote" { +@test "init guard renders the real remote URL in join guidance when .secrets-store carries a remote" { mkdir -p "$HOME/.secrets-work" age-keygen -o "$HOME/.secrets-work/key.txt" 2>/dev/null [ -s "$HOME/.secrets-work/key.txt" ] @@ -1660,7 +1661,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] || false + [[ "$output" == *"secrets join --remote git@example.com:me/secrets-work.git"* ]] || false } # ─── EGB-652: `file` external type (whole-file sync, e.g. Android keystore) ── From 7b041af68b2e13ee4c1281a6be3a4463def14297 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 16:23:50 -0700 Subject: [PATCH 45/70] feat: thin install.sh onboarding bootstrap (EGB-671) Ships in the repo (clone already done). Checks age + jq + git, then PRINTS the PATH line, onboarding next-steps, upgrade one-liner, and key-transfer hint. Never edits shell rc, never runs sudo (prints the command). Exits non-zero with an install hint when a dependency is missing. --- install.sh | 113 ++++++++++++++++++++++++++++++++++++++++++++++ test/install.bats | 72 +++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100755 install.sh create mode 100644 test/install.bats diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..695fc59 --- /dev/null +++ b/install.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# +# secrets — thin onboarding bootstrap (EGB-671). +# +# This script ships INSIDE the repo: you already cloned the repo to get it, so +# its only jobs are (1) verify the dependencies the tool needs and (2) print the +# exact commands to finish setup. It deliberately does NOT: +# - edit your shell rc files (it prints the PATH line for you to paste) +# - invoke sudo or install packages behind your back (it prints the command) +# - re-implement any of the tool's security logic +# +# This is a security tool whose whole pitch is "verify, don't trust" — so the +# installer holds itself to a higher bar than convenience, not a lower one. +# +# Usage: +# ./install.sh # check deps, print setup + next steps +# ./install.sh --help + +set -euo pipefail + +# Resolve the directory this script lives in (the cloned tool repo). Uses bash +# builtins only so it works under a minimal PATH. +_src="${BASH_SOURCE[0]}" +TOOL_DIR="$(cd "${_src%/*}" 2>/dev/null && pwd)" + +usage() { + cat < + Other machine: secrets join --remote --key +EOF +} + +# Print the install command for a package, using whatever package manager is +# present. For sudo-requiring managers we PRINT the line for you to run — the +# installer never escalates on its own. +install_hint() { + local pkg="$1" + if command -v brew >/dev/null 2>&1; then + echo "brew install $pkg" + elif command -v apt-get >/dev/null 2>&1; then + echo "sudo apt-get install -y $pkg" + elif command -v dnf >/dev/null 2>&1; then + echo "sudo dnf install -y $pkg" + else + echo "install '$pkg' with your system package manager" + fi +} + +case "${1:-}" in + --help|-h) usage; exit 0 ;; + "") ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; +esac + +echo "secrets — bootstrap check (tool dir: $TOOL_DIR)" +echo "" + +# Dependency check. age + jq + git are all load-bearing on the cold-start path: +# jq became required once .secrets.json (manifest) is JSON, so it must be present +# BEFORE the first manifest read. +missing=0 +for dep in git age jq; do + if command -v "$dep" >/dev/null 2>&1; then + echo " ok $dep" + else + echo " MISSING $dep — install it with:" + echo " $(install_hint "$dep")" + missing=1 + fi +done +echo "" + +if [ "$missing" -ne 0 ]; then + echo "Install the missing dependencies above, then re-run ./install.sh." >&2 + exit 1 +fi + +cat < + # then transfer key.txt to your other machines (AirDrop / scp / USB): + # scp :$HOME/.secrets/key.txt ~/.secrets/key.txt + + Other machine (join an existing vault): + secrets join --remote --key + # 'join' clones the vault, installs the key, and VERIFIES it decrypts + # before declaring success — a mis-copied key fails loudly, not silently. + +To update the tool later: + git -C "$TOOL_DIR" pull +EOF diff --git a/test/install.bats b/test/install.bats new file mode 100644 index 0000000..0ce7593 --- /dev/null +++ b/test/install.bats @@ -0,0 +1,72 @@ +#!/usr/bin/env bats +# EGB-671: install.sh thin bootstrap. It ships IN the repo (you clone the repo +# to get it), so its job is: verify deps (age + jq + git), PRINT the PATH line +# and next-step commands — never edit dotfiles, never invoke sudo. Security-rail +# concerns are operator-local (.ship-policy.json); these are functional checks. + +load test_helper + +INSTALL_SH="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)/install.sh" + +@test "install.sh exists and is executable" { + [ -f "$INSTALL_SH" ] + [ -x "$INSTALL_SH" ] +} + +@test "install.sh --help prints usage and exits 0" { + run "$INSTALL_SH" --help + [ "$status" -eq 0 ] + [[ "$output" == *"install.sh"* ]] || false + [[ "$output" == *"join"* ]] || false +} + +@test "install.sh prints the PATH export line for the tool dir (does not edit rc)" { + local tool_dir + tool_dir="$(cd "$(dirname "$INSTALL_SH")" && pwd)" + run "$INSTALL_SH" + [ "$status" -eq 0 ] + [[ "$output" == *"export PATH="* ]] || false + [[ "$output" == *"$tool_dir"* ]] || false + # It must NOT have written to any shell rc in the isolated HOME. + [ ! -f "$HOME/.zshrc" ] + [ ! -f "$HOME/.bashrc" ] +} + +@test "install.sh prints both onboarding next-steps (init --remote and join)" { + run "$INSTALL_SH" + [ "$status" -eq 0 ] + [[ "$output" == *"secrets init --remote"* ]] || false + [[ "$output" == *"secrets join --remote"* ]] || false +} + +@test "install.sh prints the upgrade one-liner" { + run "$INSTALL_SH" + [ "$status" -eq 0 ] + [[ "$output" == *"git -C"* ]] || false + [[ "$output" == *"pull"* ]] || false +} + +@test "install.sh prints a key-transfer hint" { + run "$INSTALL_SH" + [ "$status" -eq 0 ] + [[ "$output" == *"key.txt"* ]] || false +} + +@test "install.sh never invokes sudo (prints it for the user instead)" { + # No executed 'sudo' — any sudo reference must be quoted guidance text. + run grep -nE '^[[:space:]]*sudo ' "$INSTALL_SH" + [ "$status" -ne 0 ] +} + +@test "install.sh reports a missing dependency with an install hint and non-zero exit" { + # Build a minimal PATH that has the tools install.sh needs but NOT jq. + local fake="$TEST_TMPDIR/fakebin" + mkdir -p "$fake" + for t in bash uname env cat grep sed tr dirname command age git printf; do + src="$(command -v "$t" 2>/dev/null || true)" + [ -n "$src" ] && ln -sf "$src" "$fake/$t" 2>/dev/null || true + done + run env PATH="$fake" "$INSTALL_SH" + [ "$status" -ne 0 ] + [[ "$output" == *"jq"* ]] || false +} From 4d975d447df5913a6869a2567c561cfa8db1dbe1 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Mon, 8 Jun 2026 16:23:50 -0700 Subject: [PATCH 46/70] chore: rewrite onboarding docs + bump version (v0.7.3.0) README rewritten to the install.sh + init --remote + join flow; dropped the macOS-only prerequisite (age+jq install hints now cover apt/dnf too). CHANGELOG entry for EGB-671. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 36 +++++++++++++++ README.md | 122 ++++++++++++++++++++++++++++----------------------- VERSION | 2 +- 3 files changed, 104 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a51e6a..23e6122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.3.0] - 2026-06-08 + +### Added + +- **Real install / onboarding scripts (EGB-671)** — onboarding a machine is now + (close to) one command, and a mis-copied key fails loudly instead of silently. + - **`secrets join --remote --key `** — second-machine onboarding in + one verb: clones the vault, installs the key at mode 600, and **verifies the + key actually decrypts the store before declaring success**. An empty vault + reports "nothing to verify yet" (it never prints a false `VERIFIED`); a wrong + key fails loudly with the store left in place to fix. All security logic + (store resolution, URL handling, path rails) is reused from the audited core, + not re-implemented in a side script. + - **`secrets init --remote `** — wires the remote and pushes the initial + store so the upstream branch exists, so your first project `push` doesn't trip + the fast-forward-pull guard on a brand-new empty remote. Run interactively, + `init` also offers to add your first project's secrets (default No, skipped + under `--yes` / non-interactive, so it stays a clean primitive for CI). + - **`install.sh`** — thin bootstrap that ships in the repo: checks `age` + `jq` + + `git`, then prints the `PATH` line, the onboarding next-steps, the upgrade + one-liner, and a key-transfer hint. It never edits your shell config and never + runs `sudo` (it prints the command so you stay in control). + - **First-manifest `options.autoAdd` prompt (EGB-677 contract #2)** — the first + `push` that scaffolds a project's manifest now records an explicit, committed + `options.autoAdd` value (asked once when interactive; the default ON, written + explicitly, under automation). + +### Fixed + +- **Day-2 silent decrypt failure** — `secrets pull` now dies loudly when a blob + fails to decrypt with the current key (all three decrypt paths), instead of + emitting a warning and continuing with exit 0. A wrong key can no longer pass + unnoticed after onboarding. +- The `secrets init` second-machine trap now points at `secrets join` (the real + one-command path) instead of a manual `git clone`. + ## [0.7.2.0] - 2026-06-08 ### Added diff --git a/README.md b/README.md index e345d64..4fa3459 100644 --- a/README.md +++ b/README.md @@ -58,83 +58,95 @@ Beyond project files, `secrets` can also sync files that live *outside* the proj ## Prerequisites -- **macOS** (uses Homebrew for installation) -- **git** (already installed on most Macs — type `git --version` to check) -- **age** (the encryption tool — installed in step 1 below) +- **macOS or Linux** +- **git** (`git --version` to check) +- **age** and **jq** — `install.sh` checks for these and prints the exact install command for your platform (Homebrew on macOS, `apt`/`dnf` on Linux) ## Setup -### First machine (one-time setup) +Clone the tool repo, then run `install.sh`. It checks dependencies and prints the +two commands to finish setup. It never edits your shell config and never runs +sudo — it prints the commands so you stay in control. ```bash -# 1. Install the encryption tool -brew install age - -# 2. Download the secrets tool (this repo — contains only the CLI, no secret files) git clone https://codeberg.org/egbt/secrets.git ~/dev/secrets - -# 3. Make the 'secrets' command available everywhere -# Add this line to your shell config file (~/.zshrc on Mac): -export PATH="$HOME/dev/secrets:$PATH" -# Then restart your terminal, or run: -source ~/.zshrc - -# 4. Initialize your encrypted secrets store -# This creates a folder at ~/.secrets/ with your encryption key -secrets init - -# 5. Create a PRIVATE repository on GitHub to store your encrypted secrets -# Go to github.com/new, name it something like 'my-secrets', and make sure -# "Private" is selected. Then connect it: -cd ~/.secrets -git remote add origin git@github.com:/my-secrets.git -git push -u origin main +cd ~/dev/secrets +./install.sh ``` -> **Important:** Step 5 creates a *separate* private repo for your encrypted secrets. This is different from the `secrets` tool repo you cloned in step 2. The tool repo can be public — it contains no secrets. The `~/.secrets/` repo must be private. +`install.sh` prints a `export PATH="$HOME/dev/secrets:$PATH"` line — add it to your +shell config (`~/.zshrc` or `~/.bashrc`) and restart your terminal. Then onboard +this machine with one of the two flows below. -### Additional machines - -On each new machine (your desktop, a teammate's laptop, etc.): +### First machine (new vault) ```bash -# 1. Install prerequisites and the tool (same as steps 1-3 above) -brew install age -git clone https://codeberg.org/egbt/secrets.git ~/dev/secrets -export PATH="$HOME/dev/secrets:$PATH" # add to ~/.zshrc +# 1. Create a PRIVATE repo for your encrypted secrets (github.com/new or a +# Codeberg/GitLab private repo). It holds only ciphertext — never your key. +# Then wire it up and push the store in one command: +secrets init --remote git@github.com:/my-secrets.git -# 2. Clone the encrypted secrets repo -git clone git@github.com:/my-secrets.git ~/.secrets - -# 3. Copy the encryption key from your first machine -# This is the only step that requires direct machine-to-machine transfer. -# Choose one method: -# -# Option A: AirDrop (Mac to Mac) -# On your first machine, right-click ~/.secrets/key.txt → Share → AirDrop -# Save it to ~/.secrets/key.txt on the new machine -# -# Option B: Secure copy over SSH -# scp first-machine:~/.secrets/key.txt ~/.secrets/key.txt -# -# Option C: USB drive -# Copy key.txt to a USB drive, transfer it, delete from USB after - -# 4. Pull your secrets into any project +# 2. (optional) Add a project's secrets. From a project directory: cd ~/myapp -secrets pull +secrets push +# The first push asks once whether to auto-track new env files and records +# your choice in the project's .secrets.json. ``` +`secrets init --remote` generates your key (`~/.secrets/key.txt`), wires the +remote, and pushes the initial store so the upstream branch exists. The private +secrets repo is separate from this tool repo — the tool repo is public and holds +no secrets; the `~/.secrets/` repo must be private. + +> Running `secrets init` interactively (in a terminal) also offers to add your +> first project's secrets right away. Run it with `--yes` (or in any non-tty +> context like CI) to skip that prompt and just create the vault. + +### Other machines (join an existing vault) + +On a second machine, a desktop, or a teammate's laptop: + +```bash +# 1. Clone the tool and run the bootstrap (as in Setup above) +git clone https://codeberg.org/egbt/secrets.git ~/dev/secrets +cd ~/dev/secrets && ./install.sh # add the printed PATH line to your shell config + +# 2. Get key.txt onto this machine (the one manual, out-of-band step): +# AirDrop (Mac→Mac), or +# scp first-machine:~/.secrets/key.txt ~/Downloads/key.txt, or +# a USB drive (delete from the drive afterward) + +# 3. Join the vault in one command: +secrets join --remote git@github.com:/my-secrets.git --key ~/Downloads/key.txt +``` + +`secrets join` clones the vault, installs the key at mode 600, and **verifies the +key actually decrypts the store before declaring success** — a mis-copied key +fails loudly here, not silently on a later `secrets pull`. On success it tells you +to run `secrets pull` in any project. + > **The key file (`~/.secrets/key.txt`) is the only thing that needs to be transferred manually.** It never leaves your machines — it's excluded from git, never uploaded, never transmitted over the internet. Anyone with this file can decrypt all your secrets, so treat it like a password. ### Sharing with teammates To share secrets with a teammate, they need: -1. Access to your private `my-secrets` GitHub repo (add them as a collaborator) -2. A copy of `key.txt` (send it to them directly — AirDrop, USB, or in-person) +1. Access to your private secrets repo (add them as a collaborator) +2. A copy of `key.txt` (send it directly — AirDrop, USB, or in-person) -Everyone on the team uses the same key. When anyone runs `secrets push`, the encrypted files are updated and everyone else can `secrets pull` to get the latest version. +Everyone on the team uses the same key. A teammate joins with +`secrets join --remote --key `. When anyone runs +`secrets push`, the encrypted files update and everyone else runs `secrets pull` +to get the latest. + +### Updating the tool + +```bash +git -C ~/dev/secrets pull +``` + +If your store was last written by a newer client than yours, `secrets` prints a +one-line version-skew nudge — that's your cue to run the command above. ## Usage diff --git a/VERSION b/VERSION index 9872478..934346d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.2.0 +0.7.3.0 From 1ee096cc3303530dde3265fe7d36d4767c5ff0b6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Thu, 18 Jun 2026 10:08:13 -0700 Subject: [PATCH 47/70] refactor: dedup external extractor + read guards, warn on legacy-pull nested blobs (EGB-701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EGB-677 stage-1 structural cleanups, no behavior change for the v2 happy path: 1. cmd_which reuses _json_external_entries (the push/pull extractor) instead of its own duplicated jq @tsv projection, so `which` applies the same properties->gradle-properties normalization + skip-with-warning rules the sync path does and can't drift from it. 2. The two external-manifest read guards are factored into _json_readable (plain regular file, silent) / _legacy_readable (warn+skip a symlinked .secrets-files), shared by _external_entries_for_push/_pull. 3. The legacy (manifest-less) pull path now warns when nested /.age blobs exist that its non-recursive globs can't see (external/ excluded — pull_external_files handles those), so it never silently under-restores. Tests: +4 in test/manifest.bats (normalized which display, malformed external skipped by which, nested-blob warning fires, external-only no false warning). Full suite green (286/286). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 6 ++-- secrets | 84 +++++++++++++++++++++++++++++++++++----------- test/manifest.bats | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 608040a..bd2954f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -76,8 +76,8 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (140 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (78 tests) - migrate.bats # EGB-703 store-format-v2 migration tests (26 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (83 tests) + migrate.bats # EGB-703 store-format-v2 migration tests (35 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file @@ -107,7 +107,7 @@ The active store directory is picked by `resolve_store()` using these rules, hig Key design decisions (all driven by /autoplan review): - **Wire-in is at command scope** (`cmd_push`/`cmd_pull`), via `push_external_files` / `pull_external_files`, **not** inside `push_dir_to_project` / `pull_project_to_dir` (those loop per-workspace and `pull_project_to_dir` uses stdout as a data channel). -- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). `cmd_list --json` (EGB-699) emits the same recursive walk as a machine-readable object (`{store, projects[].entries[]}`, each entry `dotenv`→`path` or `external`→`subtype`+`path`) for tooling/CI (feeds EGB-671); jq assembles it so paths escape correctly and stdout stays pure JSON (the human store hint is suppressed; jq is a hard dep only in `--json` mode). +- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). **EGB-701 cleanups:** (1) the *legacy* (manifest-less) `pull` keeps its non-recursive globs but now **warns** when nested `/.age` blobs exist that those globs can't see (it excludes `external/`, which `pull_external_files` handles) — so a manifest-less pull never silently under-restores; the fix the warning points at is committing a `.secrets.json`. (2) `cmd_which`, push, and pull share one external extractor (`_json_external_entries`), so `which` applies the same `properties`→`gradle-properties` normalization and skip-with-warning rules the sync path does (it shows exactly what will sync, not a stale raw projection). (3) the two external-manifest read guards are factored into `_json_readable` (plain regular file, silent) / `_legacy_readable` (warn-and-skip on a symlinked legacy manifest). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). `cmd_list --json` (EGB-699) emits the same recursive walk as a machine-readable object (`{store, projects[].entries[]}`, each entry `dotenv`→`path` or `external`→`subtype`+`path`) for tooling/CI (feeds EGB-671); jq assembles it so paths escape correctly and stdout stays pure JSON (the human store hint is suppressed; jq is a hard dep only in `--json` mode). - **Merge is pure bash, no `sed`/regex** (`merge_gradle_keys`): exact-string key comparison (avoids `beaconClerkPk` vs `beaconClerkPkTest` substring bug), value treated as opaque literal (survives `& \ /` in values). Updates a managed key in place at its first occurrence, collapses duplicates, appends new keys, preserves unrelated lines/comments/order. Continuation lines (trailing odd backslashes, tracked by `_trailing_bs_odd`) are never matched as keys. Atomic write: temp in the same dir → `chmod` to match (or `600` on create) → `mv`. Backs up to `.secrets-bak` before each merge. - **Properties separator parsing** (`_props_get`): key ends at the first `=`, `:`, or whitespace (after lstrip); handles `key=value`, `key = value`, `key:value`, `key value`; last definition wins. - **Security:** the write target comes from a committed file, so `_validate_external_target_path` locks it down — basename must be `gradle.properties`, must resolve inside `$HOME` (deepest-existing-ancestor resolved, symlink target/parent refused, `..` rejected). This blocks a malicious manifest from appending decrypted keys to `~/.gitconfig`/`~/.bashrc`. `_parse_secrets_files_manifest` rejects shell metacharacters/control chars in path and keys (path allows `[A-Za-z0-9/._~-]` only; keys allow `[A-Za-z0-9._-]` + space), mirrors the `.secrets-store` posture (no shell expansion, symlinked manifest skipped). diff --git a/secrets b/secrets index 219cfbf..2d55aa7 100755 --- a/secrets +++ b/secrets @@ -1038,6 +1038,31 @@ _json_external_entries() { done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$manifest") } +# EGB-701 item 2: the read guards for the two external-manifest sources, +# factored out of _external_entries_for_push/_pull so they can't drift. +# +# _json_readable — true when a .secrets.json is a safe regular file to read. +# A symlinked manifest is treated as absent and silently ignored: it's the +# project's own committed file, so a symlink there is just skipped (the fatal +# symlink refusal lives in _check_manifest_file, used by the linting paths). +_json_readable() { + [ -f "$1" ] && [ ! -L "$1" ] +} + +# _legacy_readable — true when a legacy .secrets-files is a safe regular file +# to read, warning (and returning false) when it exists but is a symlink: a +# symlinked legacy manifest's target is attacker-influenceable, so never follow +# it. A missing or non-regular file returns false silently. +_legacy_readable() { + local legacy="$1" + [ -e "$legacy" ] || return 1 + if [ -L "$legacy" ]; then + echo "WARNING: $legacy is a symlink; ignoring." >&2 + return 1 + fi + [ -f "$legacy" ] +} + # External tuples for PUSH: .secrets.json entries first, then legacy # .secrets-files entries whose (type, path) the manifest doesn't cover — # the absorb set, which cmd_push folds into the manifest after a @@ -1046,23 +1071,19 @@ _external_entries_for_push() { local root="$1" local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" local seen="" t p k - if [ -f "$json" ] && [ ! -L "$json" ]; then + if _json_readable "$json"; then while IFS=$'\t' read -r t p k; do [ -n "$t" ] || continue printf '%s\t%s\t%s\n' "$t" "$p" "$k" seen="$seen$t|$p"$'\n' done < <(_json_external_entries "$json") fi - if [ -e "$legacy" ]; then - if [ -L "$legacy" ]; then - echo "WARNING: $legacy is a symlink; ignoring." >&2 - elif [ -f "$legacy" ]; then - while IFS=$'\t' read -r t p k; do - [ -n "$t" ] || continue - case "$seen" in *"$t|$p"$'\n'*) continue ;; esac - printf '%s\t%s\t%s\n' "$t" "$p" "$k" - done < <(_parse_secrets_files_manifest "$legacy") - fi + if _legacy_readable "$legacy"; then + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + case "$seen" in *"$t|$p"$'\n'*) continue ;; esac + printf '%s\t%s\t%s\n' "$t" "$p" "$k" + done < <(_parse_secrets_files_manifest "$legacy") fi } @@ -1071,19 +1092,18 @@ _external_entries_for_push() { _external_entries_for_pull() { local root="$1" local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" - if [ -f "$json" ] && [ ! -L "$json" ]; then - if [ -f "$legacy" ] && [ ! -L "$legacy" ]; then + if _json_readable "$json"; then + # A regular (non-symlink) legacy file alongside the manifest is superseded: + # warn but don't read it. _json_readable is the "plain regular file" test — + # exactly the supersede condition (and unlike _legacy_readable it stays + # silent on a symlink, matching the original no-warn-on-symlink behavior). + if _json_readable "$legacy"; then echo "WARNING: $legacy is superseded by $SECRETS_JSON_NAME and was ignored on pull. Run 'secrets push' to absorb it, then delete it." >&2 fi _json_external_entries "$json" return 0 fi - [ -e "$legacy" ] || return 0 - if [ -L "$legacy" ]; then - echo "WARNING: $legacy is a symlink; ignoring." >&2 - return 0 - fi - [ -f "$legacy" ] && _parse_secrets_files_manifest "$legacy" + _legacy_readable "$legacy" && _parse_secrets_files_manifest "$legacy" return 0 } @@ -1838,6 +1858,26 @@ cmd_pull() { info "Decrypted $count file(s) into $target_dir" + # EGB-701 item 3: the globs above are non-recursive, so a nested dotenv blob + # (/.age) written by a manifest-driven push on another + # machine is invisible here — silently restored nothing, counted nothing. + # external/.age blobs are restored by pull_external_files, so exclude + # them. Warn (don't die) so a manifest-less pull never under-restores in + # silence; the fix is a committed .secrets.json, which the recursive + # manifest-driven branch above handles correctly. + local nested + nested=$(find "$SECRETS_DIR/$project" -mindepth 2 -type f -name '*.age' \ + -not -path "$SECRETS_DIR/$project/external/*" 2>/dev/null) + if [ -n "$nested" ]; then + echo "WARNING: this project has nested encrypted files the manifest-less pull can't restore:" >&2 + while IFS= read -r nf; do + [ -n "$nf" ] || continue + local rel="${nf#"$SECRETS_DIR/$project/"}" + echo " ${rel%.age}" >&2 + done <<< "$nested" + echo " Add a $SECRETS_JSON_NAME manifest (run 'secrets push' on a machine that has these files) so they restore." >&2 + fi + # Merge any external files (.secrets-files) declared in this project. pull_external_files "$PWD" "$project" @@ -2307,11 +2347,15 @@ cmd_which() { echo " dotenv $entry [UNSAFE — will be refused]" fi done < <(jq -r '.dotenv // [] | .[]' "$json_manifest") + # EGB-701 item 1: reuse the one external extractor the sync path uses, + # so `which` applies the same normalization + skip-with-warning rules + # push/pull do — `which` shows exactly what will sync, never a stale + # raw projection that drifts from the helper. local etype epath ekeys while IFS=$'\t' read -r etype epath ekeys; do [ -n "$etype" ] || continue echo " $etype $epath $ekeys" - done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$json_manifest") + done < <(_json_external_entries "$json_manifest") fi # Read back any external-file manifest in cwd (validates the format and diff --git a/test/manifest.bats b/test/manifest.bats index ce16125..f7aeab9 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -642,6 +642,31 @@ m_nojq_path() { [[ "$output" == *"k1"* ]] || false } +# EGB-701 item 1: `which` and the push/pull external extractor share one +# helper (_json_external_entries), so `which` applies the same +# properties→gradle-properties normalization the sync path uses — no drift. +@test "which normalizes a properties external to the gradle-properties token (EGB-701)" { + create_project_dir whichnorm + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["k1"]}]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"gradle-properties"* ]] || false +} + +# EGB-701 item 1: a malformed external (a properties entry with no keys) is +# skipped by the sync path; routing `which` through the shared extractor means +# `which` skips+warns it too, so it faithfully shows what actually syncs +# rather than printing an entry push/pull silently drop. +@test "which skips a malformed external entry the sync path would drop (EGB-701)" { + create_project_dir whichmalformed + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties"}]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"has no keys"* ]] || false + # The skipped entry's path must NOT appear in the printed manifest summary. + [[ "$output" != *" gradle-properties ~/.gradle/gradle.properties"* ]] || false +} + # ─── F: ship Step 7 coverage backfill (audit gaps) ───────────────────── @test "which flags an unsafe dotenv entry with the UNSAFE marker" { @@ -739,6 +764,46 @@ m_nojq_path() { [ "$(cat packages/web/.env.development)" = "N=nested" ] } +@test "legacy (manifest-less) pull warns about nested blobs it can't restore (EGB-701)" { + # The legacy pull path globs only top-level *.age/.*.age. A nested dotenv + # blob (/.age) written by a manifest-driven push on another + # machine is invisible to those globs — restored nothing, counted nothing. + # The fix: warn so a manifest-less pull never silently under-restores. + init_with_remote + create_project_dir nestlegacy + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + [ -f "$SECRETS_DIR/nestlegacy/packages/web/.env.development.age" ] + # Simulate a machine with no manifest: drop .secrets.json + local files, + # forcing the legacy non-recursive glob branch. + rm -f .secrets.json + rm -rf packages + run "$SECRETS_BIN" pull nestlegacy + [ "$status" -eq 0 ] + # The warning names the nested blob and points at the manifest as the fix. + [[ "$output" == *"packages/web/.env.development"* ]] || false + [[ "$output" == *"$SECRETS_JSON_NAME"* || "$output" == *".secrets.json"* ]] || false + # The legacy path genuinely can't restore it (the warning is the contract). + [ ! -f packages/web/.env.development ] +} + +@test "legacy pull does NOT warn about external/ blobs (handled separately, EGB-701)" { + # external/.age blobs are restored by pull_external_files, not the + # dotenv globs, so they must not trip the nested-blob warning. + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir extnolwarn + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + [ -d "$SECRETS_DIR/extnolwarn/external" ] + run "$SECRETS_BIN" pull extnolwarn + [ "$status" -eq 0 ] + [[ "$output" != *"can't restore"* ]] || false + [[ "$output" != *"nested encrypted"* ]] || false +} + @test "list shows a nested manifest blob" { init_with_remote create_project_dir nestlist From 3ece393cc5ada221964976648201216b8174facd Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Thu, 18 Jun 2026 10:08:13 -0700 Subject: [PATCH 48/70] chore: bump version and changelog (v0.7.3.1) Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e6122..475794a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.3.1] - 2026-06-18 + +### Changed + +- **EGB-677 stage-1 structural cleanups (EGB-701)** — tech-debt dedup with one + new safety warning; no behavior change for the manifest-driven (v2) happy path. + - **`secrets which` now reuses the one external-entry extractor** the push/pull + path uses (`_json_external_entries`) instead of its own duplicated `jq` + projection. So `which` applies the same `properties`→`gradle-properties` + normalization and skips (with a warning) the same malformed external entries + the sync path drops — `which` shows exactly what will sync, not a stale raw + projection that could drift from the real behavior. + - **The two external-manifest read guards are factored into shared helpers** — + `_json_readable` (plain regular file, silent) and `_legacy_readable` (warns + and skips a symlinked `.secrets-files`) — so `_external_entries_for_push` and + `_external_entries_for_pull` can't drift apart. + +### Fixed + +- **Legacy (manifest-less) `pull` no longer silently under-restores (EGB-701)** — + the manifest-less pull path globs only top-level `*.age`/`.*.age`, so a nested + dotenv blob (`/.age`) written by a manifest-driven push on + another machine was invisible: restored nothing, counted nothing, said nothing. + It now **warns** and names each nested blob it can't reach (external blobs are + excluded — `pull_external_files` handles those), pointing at committing a + `.secrets.json` as the fix. The manifest-driven pull already restored nesting + correctly; this only closes the legacy path's blind spot. + ## [0.7.3.0] - 2026-06-08 ### Added diff --git a/VERSION b/VERSION index 934346d..512d674 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3.0 +0.7.3.1 From 09c4ad54f9be7383dce2d2884e6483a1ef459910 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Thu, 18 Jun 2026 11:01:17 -0700 Subject: [PATCH 49/70] =?UTF-8?q?feat:=20secrets=20upgrade=20verb=20?= =?UTF-8?q?=E2=80=94=20self-update=20+=20re-check=20version=20skew=20(EGB-?= =?UTF-8?q?716)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pairs the EGB-713 skew WARNING with a fix path. `secrets upgrade` fast-forwards the tool's own git checkout (git -C "$SCRIPT_DIR" pull --ff-only; never merges or rewrites local commits), reports vOLD -> vNEW, then best-effort re-checks the store's recorded writer-version against the new version so the operator sees whether the nudge is cleared. `secrets upgrade --check` reports availability without pulling. Thin and explicit: no auto-update, no background polling (security tool). Directed errors for not-a-checkout / no-upstream / diverged / offline. cmd_upgrade never calls check_initialized (it's about the tool, not the store); the skew re-check is silent unless a store with a writer-version resolves. Wired into the dispatcher (upgrade) shift; cmd_upgrade "$@") and cmd_help. Tests: test/upgrade.bats (8) run a relocated script copy in a throwaway git repo with a bare upstream, so the real checkout is never touched. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 5 +- README.md | 2 + secrets | 87 +++++++++++++++++++++++++++++++++++ test/upgrade.bats | 115 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 test/upgrade.bats diff --git a/CLAUDE.md b/CLAUDE.md index bd2954f..9693fde 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,13 +56,13 @@ skips security specialist + red team, and Step 11 skips adversarial review. ## Architecture -Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify, migrate. +Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify, migrate, upgrade. - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). Store layout: nested dotenv entries land at `/.age` (relpath preserved — the store self-describes where a file restores). jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. -- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker. **Additive v2 (EGB-712):** reads resolve a `properties` blob by trying `.properties.age` then falling back to `.gradle-properties.age` (`_resolve_external_blob_read`); writes dual-write a `properties` external only when a v1 twin already exists in the store (`_external_blob_write_targets`), so existing externals keep old clients fresh while brand-new externals are written v2-only (a gentle forcing function). Blob location no longer depends on the marker — the old `_external_blob_suffix` is gone. `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE, plus a `v2-only` count of externals old clients can't read), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710/EGB-712). **Under additive v2 (EGB-712) `--finalize` is now OPTIONAL GC, not a required milestone:** because upgraded clients dual-write existing externals and read-fall-back, *not* finalizing never cuts anyone off — finalize only reclaims the duplicate v1 blobs and stays deferrable indefinitely (defusing the cross-machine coordination gate). dotenv and `file` blobs are identical across formats, so they always propagate to old clients; only a brand-new `properties` external is v2-only. **Version-skew nudge (EGB-713):** a committed `$SECRETS_DIR/.secrets-writer-version` records the highest client `VERSION` that has written to the store (monotonic; stamped via `_stamp_writer_version` right before each store-committing `git add -A` — push/rekey/migrate/finalize — never on read paths, so it always rides a commit and never dangles to break `pull --ff-only`). `check_initialized` calls `_check_store_version_skew`, which warns once per invocation (stderr, non-fatal, `set -e`-safe) when the store's stamp is numerically greater than `_client_version` (read from `$SCRIPT_DIR/VERSION`); `secrets which` prints the `written-by:` line. Stores with no stamp (pre-EGB-713) are silent. The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. +- Store format (EGB-677 stage 2 / EGB-703): the store is self-describing via a committed one-line `$SECRETS_DIR/.secrets-format` file (`2`). Absence ⇒ v1 (every store predating EGB-703). v2's only on-disk change vs v1 is the external `properties` blob suffix: `.gradle-properties.age` → `.properties.age` (matching the manifest `type`); dotenv and `file` blobs are unchanged. `_store_format()` reads the marker. **Additive v2 (EGB-712):** reads resolve a `properties` blob by trying `.properties.age` then falling back to `.gradle-properties.age` (`_resolve_external_blob_read`); writes dual-write a `properties` external only when a v1 twin already exists in the store (`_external_blob_write_targets`), so existing externals keep old clients fresh while brand-new externals are written v2-only (a gentle forcing function). Blob location no longer depends on the marker — the old `_external_blob_suffix` is gone. `init` stamps a fresh store v2 (born-v2). `secrets which` prints the store-format line `format: vN`, and (EGB-700) when a `.secrets.json` is present the manifest header line also carries its schema version (`manifest (.secrets.json at , version N):`). **Migration is copy-forward and non-destructive:** `secrets migrate --dry-run` (per project, reports old→new, writes nothing) → `secrets migrate` (per project, manifest-free: enumerates the store's `*.gradle-properties.age` blobs directly — same source of truth as `--finalize` — and writes their `.properties.age` twins, so a legacy `.secrets-files`-only project with no `.secrets.json` migrates cleanly and no store blob is left un-twinned; idempotent; EGB-710) → `secrets migrate --finalize` (store-wide; the ONLY destructive step — gates on `verify --all` green + every v1 blob having a v2 twin, cuts a `pre-v2-migrate-` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation since a lagging v1 client against a finalized store stops seeing `properties` externals until it upgrades). `secrets migrate --status` is a read-only survey that walks every project in the store and reports each one's v2 readiness (v2-ready / migrated / NEEDS MIGRATE, plus a `v2-only` count of externals old clients can't read), exiting non-zero while any v1 blob is un-twinned so it gates the path to `--finalize` (EGB-710/EGB-712). **Under additive v2 (EGB-712) `--finalize` is now OPTIONAL GC, not a required milestone:** because upgraded clients dual-write existing externals and read-fall-back, *not* finalizing never cuts anyone off — finalize only reclaims the duplicate v1 blobs and stays deferrable indefinitely (defusing the cross-machine coordination gate). dotenv and `file` blobs are identical across formats, so they always propagate to old clients; only a brand-new `properties` external is v2-only. **Version-skew nudge (EGB-713):** a committed `$SECRETS_DIR/.secrets-writer-version` records the highest client `VERSION` that has written to the store (monotonic; stamped via `_stamp_writer_version` right before each store-committing `git add -A` — push/rekey/migrate/finalize — never on read paths, so it always rides a commit and never dangles to break `pull --ff-only`). `check_initialized` calls `_check_store_version_skew`, which warns once per invocation (stderr, non-fatal, `set -e`-safe) when the store's stamp is numerically greater than `_client_version` (read from `$SCRIPT_DIR/VERSION`); `secrets which` prints the `written-by:` line. Stores with no stamp (pre-EGB-713) are silent. The deliberate flatten-to-basename naming the EGB-677 CEO plan sketched was dropped as lossy (it discards the restore relpath that makes the store self-describing) — see the EGB-703 eureka. **Upgrade verb (EGB-716):** `secrets upgrade` is the fix path paired with the EGB-713 skew *warning* — it `git -C "$SCRIPT_DIR" pull --ff-only`s the tool's own checkout (fast-forward only, never merges/rewrites local commits), reports `vOLD -> vNEW`, then best-effort re-checks `_store_writer_version` against the new on-disk version so the operator sees whether the nudge is cleared (the new code takes effect next invocation). `secrets upgrade --check` does `git fetch` + `rev-list --count HEAD..@{u}` and reports availability without pulling. Deliberately thin: no auto-update, no background polling (security tool). Directed errors for not-a-git-checkout / no-upstream / diverged / offline. `cmd_upgrade` never calls `check_initialized` (it's about the tool, not the store); the skew re-check is silent unless a store with a writer-version resolves. - Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` @@ -78,6 +78,7 @@ test/ secrets.bats # bats-core test suite (140 tests) manifest.bats # EGB-677 .secrets.json manifest tests (83 tests) migrate.bats # EGB-703 store-format-v2 migration tests (35 tests) + upgrade.bats # EGB-716 `secrets upgrade` self-update tests (8 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index 4fa3459..aa4662c 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ secrets clear | `secrets migrate [--dry-run]` | Copy-forward this project's encrypted blobs to store format v2 (non-destructive; manifest-free; `--dry-run` previews) | | `secrets migrate --status` | Survey every project's v2 readiness; exits non-zero until the whole store is finalize-ready | | `secrets migrate --finalize` | **Optional GC** — drop the old v1 blobs and mark the store pure v2. Never required: upgraded clients dual-write and read-fall-back, so not finalizing never cuts anyone off | +| `secrets upgrade` | Self-update the tool: `git pull --ff-only` on the `secrets` checkout, report old → new version, then re-check store version-skew. No auto-update, no background checks | +| `secrets upgrade --check` | Report whether an update is available (without pulling); changes nothing | ### Upgrading: do teammates on an older `secrets` get new secrets? diff --git a/secrets b/secrets index 2d55aa7..f567242 100755 --- a/secrets +++ b/secrets @@ -2375,6 +2375,90 @@ cmd_which() { fi } +# `secrets upgrade [--check]` (EGB-716) — self-update the TOOL checkout. +# +# Pairs the EGB-713 skew WARNING with a fix path. Deliberately thin and explicit +# (no auto-update, no background polling — this is a security tool): it only +# fast-forwards the tool's own git checkout, never merges or rewrites local +# commits. --check reports whether an update is available and changes nothing. +# After a real update it best-effort re-checks the store's writer-version skew +# against the NEW on-disk version, so the operator sees whether the EGB-713 +# nudge is now cleared (the new code itself takes effect on the next command). +cmd_upgrade() { + local check_only=0 + while [ $# -gt 0 ]; do + case "$1" in + --check) check_only=1; shift ;; + -*) die "Unknown upgrade flag: $1. Usage: secrets upgrade [--check]" ;; + *) die "Unexpected argument to upgrade: $1. Usage: secrets upgrade [--check]" ;; + esac + done + + check_cmd git + + # Self-update only works on a git checkout of the tool. + if ! git -C "$SCRIPT_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + die "secrets at $SCRIPT_DIR is not a git checkout, so it can't self-update. + Re-install by cloning the tool repo, e.g.: git clone " + fi + + # Need a tracking branch to compare against / pull from. + local upstream + upstream=$(git -C "$SCRIPT_DIR" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true) + if [ -z "$upstream" ]; then + die "No upstream tracking branch for the secrets checkout at $SCRIPT_DIR. + Set one with: git -C \"$SCRIPT_DIR\" branch --set-upstream-to=origin/main" + fi + + local oldver; oldver=$(_client_version) + + if [ "$check_only" = 1 ]; then + if ! git -C "$SCRIPT_DIR" fetch --quiet 2>/dev/null; then + die "Couldn't reach the tool remote to check for updates (offline?). + Try again when connected, or run: git -C \"$SCRIPT_DIR\" fetch" + fi + local behind; behind=$(git -C "$SCRIPT_DIR" rev-list --count "HEAD..$upstream" 2>/dev/null || echo 0) + if [ "${behind:-0}" -gt 0 ]; then + info "Update available: $behind commit(s) behind $upstream (you're on v$oldver)." + info "Apply it with: secrets upgrade" + else + info "secrets is up to date (v$oldver)." + fi + return 0 + fi + + info "Updating secrets at $SCRIPT_DIR ..." + # Fast-forward only: never merge or rewrite local commits. + if ! git -C "$SCRIPT_DIR" pull --ff-only 2>&1; then + die "Update failed (see git output above). + Likely a local change or a diverged branch in $SCRIPT_DIR. + Inspect with: git -C \"$SCRIPT_DIR\" status" + fi + + local newver; newver=$(_client_version) + if [ "$oldver" = "$newver" ]; then + info "Already up to date (v$newver)." + else + info "Upgraded: v$oldver -> v$newver" + info "The new version takes effect on your next 'secrets' command." + fi + + # Best-effort EGB-713 skew re-check against the NEW version. Silent unless a + # store with a recorded writer-version resolves. + resolve_store 2>/dev/null || true + if [ -n "${SECRETS_DIR:-}" ] && [ -f "$SECRETS_DIR/$WRITER_VERSION_FILE_NAME" ]; then + local sv; sv=$(_store_writer_version) + if [ -n "$sv" ]; then + if _version_gt "$sv" "$newver"; then + info "Note: the store was last written by v$sv — still ahead of v$newver. Another machine may run a newer client." + else + info "Your client (v$newver) is now at or ahead of the store's last writer (v$sv)." + fi + fi + fi + return 0 +} + # Decrypt-test one blob with the current key. Plaintext is streamed to # /dev/null and never written to disk (read-only contract). Returns 0 if the # blob decrypts, non-zero otherwise. @@ -2783,6 +2867,8 @@ Usage: secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` + secrets upgrade Self-update the tool (git pull --ff-only) + recheck skew + secrets upgrade --check Report whether an update is available; change nothing Tracked files: .env, .env.*, .dev.vars @@ -2952,6 +3038,7 @@ case "${1:-help}" in verify) shift; cmd_verify "$@" ;; migrate) shift; cmd_migrate "$@" ;; which|where|status) cmd_which ;; + upgrade) shift; cmd_upgrade "$@" ;; help|--help|-h) cmd_help ;; *) die "Unknown command: $1. Run 'secrets help' for usage." ;; esac diff --git a/test/upgrade.bats b/test/upgrade.bats new file mode 100644 index 0000000..041bacb --- /dev/null +++ b/test/upgrade.bats @@ -0,0 +1,115 @@ +#!/usr/bin/env bats +# EGB-716: `secrets upgrade` verb — self-update (git pull --ff-only) + skew re-check. +# +# These tests never touch the real tool checkout. Each test relocates a COPY of +# the script into a throwaway git repo wired to a bare upstream, so $SCRIPT_DIR +# (computed from BASH_SOURCE) resolves to the fake tool repo and the pull/fetch +# operate there. + +load test_helper + +# Create a fake tool repo at $TOOL (script copy + VERSION), wired to a bare +# upstream at $TOOL_REMOTE, at version $1. cd's into $TOOL (under $HOME so +# resolve_store's walk-up stays bounded and never strays to a real store). +setup_tool_repo() { + TOOL="$TEST_TMPDIR/tool" + TOOL_REMOTE="$TEST_TMPDIR/tool-remote.git" + mkdir -p "$TOOL" + cp "$SECRETS_BIN" "$TOOL/secrets" + echo "$1" > "$TOOL/VERSION" + git -c init.defaultBranch=main init -q "$TOOL" + git -C "$TOOL" add -A + git -C "$TOOL" -c user.email=t@t -c user.name=t commit -qm "v$1" + git -c init.defaultBranch=main init --bare -q "$TOOL_REMOTE" + git -C "$TOOL" remote add origin "$TOOL_REMOTE" + git -C "$TOOL" push -q -u origin HEAD:main + cd "$TOOL" +} + +# Publish a newer VERSION to the upstream (as a different clone would). +advance_tool_remote() { + local clone="$TEST_TMPDIR/tool-pub" + rm -rf "$clone" + git clone -q "$TOOL_REMOTE" "$clone" + echo "$1" > "$clone/VERSION" + git -C "$clone" -c user.email=t@t -c user.name=t commit -qam "v$1" + git -C "$clone" push -q origin HEAD:main + rm -rf "$clone" +} + +@test "upgrade --check reports an available update without changing VERSION (EGB-716)" { + setup_tool_repo 0.1.0.0 + advance_tool_remote 0.2.0.0 + run "$TOOL/secrets" upgrade --check + [ "$status" -eq 0 ] + [[ "$output" == *"Update available"* ]] || false + [[ "$output" == *"0.1.0.0"* ]] || false + # --check must not pull: local VERSION is untouched. + [ "$(cat "$TOOL/VERSION")" = "0.1.0.0" ] +} + +@test "upgrade --check is clean when already current (EGB-716)" { + setup_tool_repo 0.2.0.0 + run "$TOOL/secrets" upgrade --check + [ "$status" -eq 0 ] + [[ "$output" == *"up to date"* ]] || false +} + +@test "upgrade fast-forwards and reports old -> new (EGB-716)" { + setup_tool_repo 0.1.0.0 + advance_tool_remote 0.2.0.0 + run "$TOOL/secrets" upgrade + [ "$status" -eq 0 ] + [[ "$output" == *"v0.1.0.0 -> v0.2.0.0"* ]] || false + [ "$(cat "$TOOL/VERSION")" = "0.2.0.0" ] +} + +@test "upgrade is a no-op when already at the latest (EGB-716)" { + setup_tool_repo 0.2.0.0 + run "$TOOL/secrets" upgrade + [ "$status" -eq 0 ] + [[ "$output" == *"up to date"* ]] || false + [ "$(cat "$TOOL/VERSION")" = "0.2.0.0" ] +} + +@test "upgrade refuses when the tool dir is not a git checkout (EGB-716)" { + local d="$HOME/plain-tool" + mkdir -p "$d" + cp "$SECRETS_BIN" "$d/secrets" + echo 0.1.0.0 > "$d/VERSION" + cd "$d" + run "$d/secrets" upgrade + [ "$status" -eq 1 ] + [[ "$output" == *"git checkout"* ]] || false +} + +@test "upgrade rejects an unknown flag (EGB-716)" { + setup_tool_repo 0.1.0.0 + run "$TOOL/secrets" upgrade --bogus + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown upgrade flag"* ]] || false +} + +@test "upgrade re-checks store skew and confirms the client caught up (EGB-716)" { + setup_tool_repo 0.1.0.0 + advance_tool_remote 0.9.0.0 + # A store last written by a newer client than our starting version. + git -c init.defaultBranch=main init -q "$SECRETS_DIR" + echo 0.8.0.0 > "$SECRETS_DIR/.secrets-writer-version" + run "$TOOL/secrets" upgrade + [ "$status" -eq 0 ] + [[ "$output" == *"v0.1.0.0 -> v0.9.0.0"* ]] || false + # New client (0.9.0.0) is now ahead of the store's last writer (0.8.0.0). + [[ "$output" == *"at or ahead"* ]] || false +} + +@test "upgrade still notes when the store is ahead of the upgraded client (EGB-716)" { + setup_tool_repo 0.1.0.0 + advance_tool_remote 0.2.0.0 + git -c init.defaultBranch=main init -q "$SECRETS_DIR" + echo 0.9.0.0 > "$SECRETS_DIR/.secrets-writer-version" + run "$TOOL/secrets" upgrade + [ "$status" -eq 0 ] + [[ "$output" == *"v0.1.0.0 -> v0.2.0.0"* ]] || false + [[ "$output" == *"still ahead"* ]] || false +} From a17ae4448bc4555a0d483cde120c02c127da4bc3 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Thu, 18 Jun 2026 11:01:17 -0700 Subject: [PATCH 50/70] chore: bump version and changelog (v0.7.4.0) Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 18 ++++++++++++++++++ VERSION | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 475794a..ce00bf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.7.4.0] - 2026-06-18 + +### Added + +- **`secrets upgrade` verb (EGB-716)** — the fix path paired with the EGB-713 + version-skew *warning*. Until now the warning told you you were behind but not + how to catch up; `secrets upgrade` closes that loop. + - **`secrets upgrade`** — `git -C "$SCRIPT_DIR" pull --ff-only` on the tool's + own checkout (fast-forward only — never merges or rewrites local commits), + reports `vOLD -> vNEW`, then best-effort re-checks the store's recorded + writer-version against the new version so you see whether the EGB-713 nudge + is now cleared (the new code itself takes effect on your next command). + - **`secrets upgrade --check`** — reports whether an update is available + (`git fetch` + compare to upstream) and changes nothing. + - Deliberately thin: no auto-update, no background polling (this is a security + tool). Directed errors for not-a-git-checkout, no upstream, a diverged/dirty + branch, or being offline. + ## [0.7.3.1] - 2026-06-18 ### Changed diff --git a/VERSION b/VERSION index 512d674..584db57 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3.1 +0.7.4.0 From 9e2a563059eb2a7ed101995633672ded6a635fcd Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Wed, 24 Jun 2026 12:12:14 -0700 Subject: [PATCH 51/70] docs: multi-recipient age encryption design spec (EGB-283) Co-Authored-By: Claude Opus 4.8 (1M context) --- ...4-multi-recipient-age-encryption-design.md | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-24-multi-recipient-age-encryption-design.md diff --git a/docs/superpowers/specs/2026-06-24-multi-recipient-age-encryption-design.md b/docs/superpowers/specs/2026-06-24-multi-recipient-age-encryption-design.md new file mode 100644 index 0000000..2141dea --- /dev/null +++ b/docs/superpowers/specs/2026-06-24-multi-recipient-age-encryption-design.md @@ -0,0 +1,230 @@ +# EGB-283 — Multi-recipient age encryption + +**Date:** 2026-06-24 +**Issue:** [EGB-283](https://linear.app/egbt/issue/EGB-283) — secrets: multi-recipient age encryption (multiple keys per file) +**Status:** Design approved, ready for implementation plan +**Related:** EGB-281 (multi-store), EGB-677/EGB-703 (manifest + store-format v2) + +## Problem + +Today every blob in a store is encrypted to exactly one age public key (`age -r "$pubkey"`, +where `$pubkey` is derived from the store's single `key.txt`). The whole team shares one +private key. EGB-281's multi-store work lets users separate work/personal/client secrets into +different stores, each with its own key — but *within* a single store there is still only one key, +so onboarding/offboarding a teammate means sharing or rotating one secret by hand. + +age natively supports multiple recipients: `age -r KEY1 -r KEY2 -o file.age input` writes one +recipient stanza per key, and any matching identity decrypts. This lets a single store have N +members, each with their own keypair. Adding/removing a teammate becomes a re-encrypt against the +current recipient set — no shared password. + +## Goals + +- A store can encrypt every blob to N recipient public keys. +- Recipient set is managed with first-class commands (`secrets recipients add/rm/list`). +- The recipient set is **singular and consistent per store**: every blob is always readable by + exactly the current set. +- Fully backward compatible: existing single-key stores keep working untouched; the feature is + opt-in and detected by file presence (no store-format-marker bump). +- Decryption path is unchanged (members use their own `key.txt`). + +## Non-goals (YAGNI — explicit scope cuts) + +- **SSH recipients** (`ssh-ed25519` / `ssh-rsa`). Native age X25519 keys cover the team-key use + case; SSH adds a parsing/format axis. Clean future follow-up. +- **Per-file or per-project recipient subsets.** The whole store shares one recipient set. +- **Key discovery / distribution.** Public keys are pasted in out of band, exactly as `key.txt` + is shared today. +- **Merging recipients into a project-level config** (`.secrets.json` / `.secrets-files`). See + "Why recipients are not in the project manifest" below. + +## Design decisions (resolved during brainstorming) + +1. **Storage:** committed `recipients.txt` at the store root, managed via + `secrets recipients add/rm/list` subcommands. +2. **Re-encrypt scope:** `add`/`rm` re-encrypt the **entire store immediately** to the new set in + one commit. The store is always consistent. +3. **Backward compatibility:** absence of `recipients.txt` ⇒ exact current single-key behavior. + First `recipients add` on a legacy store bootstraps the file seeded with the local pubkey plus + the new key. `init` going forward seeds `recipients.txt` with the freshly generated pubkey + (born-multi). +4. **`rekey` semantics:** on a multi-recipient store, `rekey` becomes "re-encrypt all to the + current `recipients.txt` set" (no new keypair). On a legacy store it keeps today's behavior + (generate a new keypair, re-encrypt to it). One shared re-encrypt engine. +5. **Store config shape:** keep `recipients.txt` as its own plain, age-native file (jq-free), + alongside the existing one-line `.secrets-format` marker — matching the repo's + small-single-purpose-plain-file convention. Not folded into a JSON store-config. + +## Why recipients are not in the project manifest + +The tool has two config planes in two different git repos: + +| Plane | Location | Files | Scope | +| ----------- | -------------------------------- | -------------------------------------------------- | --------------------------- | +| **Project** | `$PWD` (the project's own repo) | `.secrets.json` (absorbs legacy `.secrets-files`), `.secrets-store` | *What this project syncs* | +| **Store** | `$SECRETS_DIR` (`~/.secrets`) | `.secrets-format`, **`recipients.txt`** (new) | *Metadata about the encrypted repo* | + +Recipients are **store-scoped** — who can decrypt *this store*, shared by every project in it. +Putting them in a project-level manifest would let each project carry its own copy and **diverge**, +the exact inconsistency the "always re-encrypt the whole store to one set" rule prevents. It also +collides with the deliberate EGB-703 decision that *the store holds no project manifest*. So the +recipient set lives with the store, next to `.secrets-format`. + +## `recipients.txt` format and security rails + +Lives at `$SECRETS_DIR/recipients.txt`, **committed** (public keys are not secret; the store +`.gitignore` only blocks `key.txt` and plaintext env files, so the file is tracked automatically). +age `-R` format: one recipient per line, `# comment` and blank lines allowed. + +``` +# alice (laptop) +age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9qsxxxxxx +# bob +age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqxxxxxx +``` + +**We do not pass the file path to `age -R`.** A committed file is an injection surface, so the +tool parses it itself into a validated indexed array `RECIPIENT_ARGS=(-r age1… -r age1…)`, +mirroring the conservative posture of `.secrets-store` / `.secrets-files`: + +- Each non-comment, non-blank line (after trim) must match a native age X25519 recipient exactly: + `^age1[0-9a-z]{58}$`. Anything else — SSH recipients, shell metacharacters, whitespace inside the + token, control/ANSI characters, `-r`-injection look-alikes — is **rejected with a clear error**. + No shell expansion, ever. +- A symlinked `recipients.txt` is refused (same rail as the manifests). +- `--name` labels (written as `# ` comment lines above the key) are restricted to + `[A-Za-z0-9 ._-]`; anything else is rejected. This blocks comment-injection into the file. +- The parser is pure bash (bash-3.2 safe). Indexed arrays are fine on bash 3.2; only *associative* + arrays are bash-4. + +Validation is the single source of truth — both the `recipients` subcommands and `_load_recipients` +(below) route through the same validator, so an externally hand-edited malicious file is caught on +the next encrypt, not just at `add` time. + +## Components + +### `_load_recipients()` — populate `RECIPIENT_ARGS` +Called once per command that encrypts. Populates the global indexed array `RECIPIENT_ARGS`: + +- `recipients.txt` present → validated array of every key in the file (error out on any invalid + line; refuse an empty/all-comment file). +- absent (legacy store) → `(-r )`, identical to today's single-recipient + behavior. + +### `_reencrypt_all()` — shared re-encrypt engine +Factored out of today's `cmd_rekey` decrypt→re-encrypt loop: + +1. Decrypt every `*.age` in the store (recursive `find -type f -name '*.age'`, covering nested + manifest dotenv blobs and `external/` blobs) with the local `key.txt` into a tmpdir. The + operator must be a current recipient; a decryption failure aborts with the old state preserved. +2. Re-encrypt each file with `age "${RECIPIENT_ARGS[@]}"` back to its relpath. +3. `ensure_store_protections`, `git add -A`, commit, push (if a remote exists). + +All recipient-changing paths call it: + +| Command | Behavior | +| ------------------------------- | -------------------------------------------------------------------- | +| `recipients add` / `rm` | edit `recipients.txt` → `_load_recipients` → `_reencrypt_all` | +| `rekey` (multi-recipient store) | `_reencrypt_all` to current set, **no new keypair** | +| `rekey` (legacy store) | today's behavior: generate new keypair, set recipients to it, re-encrypt | +| `reencrypt` (new, idempotent) | `_reencrypt_all` — heal/backfill after a manual edit | + +### `secrets recipients` subcommand +- `recipients list` — prints names + keys from `recipients.txt` (read-only, jq-free). On a legacy + store, prints the single derived pubkey with a "single-key (no recipients.txt)" note. +- `recipients add [--name