From 6dbc4e0d011b7f721d8b48d759d50c28d9ef27f9 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:21:29 -0700 Subject: [PATCH 01/12] test: make [[ ]] assertions effective under bash 3.2 (EGB-677 precursor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bats on macOS runs under system bash 3.2, where a failing [[ ]] compound command mid-test does NOT trip the ERR trap — every mid-test [[ "$output" == *...* ]] assertion in the suite was decorative. Append '|| false' to all 123 standalone [[ ]] assertion lines so failures become plain-command failures, which bats catches. Full suite still green (133/133) — no latent false positives were hiding. --- test/secrets.bats | 246 +++++++++++++++++++++++----------------------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/test/secrets.bats b/test/secrets.bats index f396171..56f051e 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -27,7 +27,7 @@ load test_helper run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"Already initialized"* ]] + [[ "$output" == *"Already initialized"* ]] || false # Key must not be overwritten local key_after @@ -51,7 +51,7 @@ load test_helper run env PATH="$fake_path" "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"age"* ]] + [[ "$output" == *"age"* ]] || false } # ─── push ────────────────────────────────────────────────────────────── @@ -73,7 +73,7 @@ load test_helper run "$SECRETS_BIN" push testproj [ "$status" -eq 1 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } @test "push errors with missing key" { @@ -83,7 +83,7 @@ load test_helper run "$SECRETS_BIN" push testproj [ "$status" -eq 1 ] - [[ "$output" == *"Key file"* ]] + [[ "$output" == *"Key file"* ]] || false } @test "push derives project name from dirname" { @@ -140,7 +140,7 @@ load test_helper run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } # ─── pull ────────────────────────────────────────────────────────────── @@ -168,7 +168,7 @@ load test_helper run "$SECRETS_BIN" pull nonexistent [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } @test "pull errors with missing key" { @@ -183,7 +183,7 @@ load test_helper run "$SECRETS_BIN" pull testproj [ "$status" -eq 1 ] - [[ "$output" == *"Key file"* ]] + [[ "$output" == *"Key file"* ]] || false } @test "pull overwrites existing files" { @@ -217,7 +217,7 @@ load test_helper run "$SECRETS_BIN" pull testproj [ "$status" -eq 0 ] [ -x "$SECRETS_DIR/.git/hooks/pre-commit" ] - [[ "$output" == *"Reinstalled"* ]] + [[ "$output" == *"Reinstalled"* ]] || false } # ─── list ────────────────────────────────────────────────────────────── @@ -231,8 +231,8 @@ load test_helper run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"projA"* ]] - [[ "$output" == *"projB"* ]] + [[ "$output" == *"projA"* ]] || false + [[ "$output" == *"projB"* ]] || false } @test "list shows empty message" { @@ -240,7 +240,7 @@ load test_helper run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"No projects"* ]] + [[ "$output" == *"No projects"* ]] || false } # ─── rm ──────────────────────────────────────────────────────────────── @@ -261,7 +261,7 @@ load test_helper run "$SECRETS_BIN" rm nonexistent [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } # ─── pre-commit hook ────────────────────────────────────────────────── @@ -275,7 +275,7 @@ load test_helper run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } @test "pre-commit allows .age files" { @@ -304,7 +304,7 @@ load test_helper run "$SECRETS_BIN" clear [ "$status" -eq 0 ] - [[ "$output" == *"Cleared 3"* ]] + [[ "$output" == *"Cleared 3"* ]] || false # Files should be gone [ ! -f "$WORK_DIR/testproj/.env" ] @@ -318,7 +318,7 @@ load test_helper run "$SECRETS_BIN" clear [ "$status" -eq 0 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } @test "clear does not remove non-secret files" { @@ -347,7 +347,7 @@ load test_helper run "$SECRETS_BIN" clear --workspaces [ "$status" -eq 0 ] - [[ "$output" == *"Cleared"* ]] + [[ "$output" == *"Cleared"* ]] || false # All should be gone [ ! -f "$mono/.env" ] @@ -368,7 +368,7 @@ load test_helper # Run a command that reads the secret run "$SECRETS_BIN" run cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false # After run completes, plaintext files should be cleared [ ! -f "$WORK_DIR/testproj/.env" ] @@ -395,7 +395,7 @@ load test_helper @test "run errors with no command" { run "$SECRETS_BIN" run [ "$status" -eq 1 ] - [[ "$output" == *"Usage"* ]] + [[ "$output" == *"Usage"* ]] || false } @test "run passes arguments through to command" { @@ -407,7 +407,7 @@ load test_helper # Run with multiple args run "$SECRETS_BIN" run ls -la .env [ "$status" -eq 0 ] - [[ "$output" == *".env"* ]] + [[ "$output" == *".env"* ]] || false } @test "run supports -- separator" { @@ -418,7 +418,7 @@ load test_helper run "$SECRETS_BIN" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false } # ─── workspaces ──────────────────────────────────────────────────────── @@ -491,7 +491,7 @@ PKGJSON run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No package.json"* ]] + [[ "$output" == *"No package.json"* ]] || false } @test "push --workspaces errors without workspaces field" { @@ -502,7 +502,7 @@ PKGJSON run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No workspaces"* ]] + [[ "$output" == *"No workspaces"* ]] || false } @test "push --workspaces errors when no env files anywhere" { @@ -517,7 +517,7 @@ EOF run "$SECRETS_BIN" push --workspaces [ "$status" -eq 1 ] - [[ "$output" == *"No secret files"* ]] + [[ "$output" == *"No secret files"* ]] || false } # ─── EGB-281: multi-store resolution ────────────────────────────────── @@ -529,8 +529,8 @@ EOF cd subdir run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "which uses .secrets-store file in cwd" { @@ -539,10 +539,10 @@ EOF create_bound_project_dir myapp "~/.secrets-work" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false # Source line must include both the rule name AND the resolved file path, # not the empty parens (".secrets-store file ()") that v0.1.0.0 shipped. - [[ "$output" == *".secrets-store file ("*"$WORK_DIR/myapp/.secrets-store)"* ]] + [[ "$output" == *".secrets-store file ("*"$WORK_DIR/myapp/.secrets-store)"* ]] || false } @test "--store flag overrides .secrets-store file and SECRETS_DIR env" { @@ -550,8 +550,8 @@ EOF create_bound_project_dir myapp "~/.secrets-from-file" run "$SECRETS_BIN" --store "$HOME/.secrets-from-flag" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-from-flag"* ]] - [[ "$output" == *"--store flag"* ]] + [[ "$output" == *"$HOME/.secrets-from-flag"* ]] || false + [[ "$output" == *"--store flag"* ]] || false } @test "which walks up to find .secrets-store in ancestor" { @@ -562,7 +562,7 @@ EOF cd "$WORK_DIR/repo/sub/deep" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "which walk-up stops at HOME boundary, does not read \$HOME/.secrets-store" { @@ -572,9 +572,9 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" != *"should-not-be-used"* ]] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" != *"should-not-be-used"* ]] || false + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "which from outside HOME falls through to default" { @@ -582,8 +582,8 @@ EOF cd /tmp run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "empty .secrets-store falls through to next rule" { @@ -593,7 +593,7 @@ EOF : > .secrets-store run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"source: default"* ]] || false } @test "comment-only .secrets-store falls through" { @@ -603,7 +603,7 @@ EOF printf '# this is a comment\n \n# another\n' > .secrets-store run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"source: default"* ]] + [[ "$output" == *"source: default"* ]] || false } @test "bare name 'work' resolves to ~/.secrets-work" { @@ -614,7 +614,7 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "~/-prefix in .secrets-store expands to HOME" { @@ -625,7 +625,7 @@ EOF cd "$WORK_DIR/repo" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-x"* ]] + [[ "$output" == *"$HOME/.secrets-x"* ]] || false } @test ".secrets-store with command injection content does not execute" { @@ -671,7 +671,7 @@ EOF run "$SECRETS_BIN" --store "$SECRETS_DIR" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"SECRET_KEY=abc123"* ]] + [[ "$output" == *"SECRET_KEY=abc123"* ]] || false } @test "uninitialized store referenced by .secrets-store gives directed error" { @@ -683,8 +683,8 @@ EOF run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone"* ]] - [[ "$output" == *"--store"* ]] + [[ "$output" == *"git clone"* ]] || false + [[ "$output" == *"--store"* ]] || false } @test "push -w ignores per-workspace .secrets-store, uses monorepo root binding" { @@ -727,7 +727,7 @@ PKG create_project_dir myapp run "$SECRETS_BIN" --store "$HOME/.secrets-work" push myapp [ "$status" -eq 0 ] - [[ "$output" == *"Store: $HOME/.secrets-work"* ]] + [[ "$output" == *"Store: $HOME/.secrets-work"* ]] || false } # ─── EGB-281: gap-filler tests (auto-decided during /ship coverage audit) ─ @@ -735,7 +735,7 @@ PKG @test "--store with missing argument errors out" { run "$SECRETS_BIN" --store [ "$status" -eq 1 ] - [[ "$output" == *"--store requires"* ]] + [[ "$output" == *"--store requires"* ]] || false } @test "--store=value (equals form) is accepted" { @@ -744,7 +744,7 @@ PKG cd "$HOME" run "$SECRETS_BIN" --store="$HOME/.secrets-equals" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-equals"* ]] + [[ "$output" == *"$HOME/.secrets-equals"* ]] || false } @test "where and status are aliases of which" { @@ -754,11 +754,11 @@ PKG cd subdir run "$SECRETS_BIN" where [ "$status" -eq 0 ] - [[ "$output" == *"source:"* ]] + [[ "$output" == *"source:"* ]] || false run "$SECRETS_BIN" status [ "$status" -eq 0 ] - [[ "$output" == *"source:"* ]] + [[ "$output" == *"source:"* ]] || false } @test "--store default sugar resolves to ~/.secrets" { @@ -766,7 +766,7 @@ PKG cd "$HOME" run "$SECRETS_BIN" --store default which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets"* ]] + [[ "$output" == *"$HOME/.secrets"* ]] || false } @test "missing key.txt in non-default store gives directed error" { @@ -783,8 +783,8 @@ PKG run "$SECRETS_BIN" push myapp [ "$status" -eq 1 ] - [[ "$output" == *"key.txt"* ]] - [[ "$output" == *"teammate"* ]] + [[ "$output" == *"key.txt"* ]] || false + [[ "$output" == *"teammate"* ]] || false } @test "CRLF line endings in .secrets-store are tolerated" { @@ -795,7 +795,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-crlf"* ]] + [[ "$output" == *"$HOME/.secrets-crlf"* ]] || false } @test "list hints at 'secrets which' when non-default store is active" { @@ -805,7 +805,7 @@ PKG run "$SECRETS_BIN" --store "$HOME/.secrets-x" list [ "$status" -eq 0 ] - [[ "$output" == *"secrets which"* ]] + [[ "$output" == *"secrets which"* ]] || false } # ─── EGB-281: adversarial-review regression tests (F1-F5) ───────────── @@ -825,7 +825,7 @@ PKG # Run a command, then verify .env is cleared by the EXIT trap run "$SECRETS_BIN" run -- cat .env [ "$status" -eq 0 ] - [[ "$output" == *"should-not-leak"* ]] + [[ "$output" == *"should-not-leak"* ]] || false # CRITICAL: the trap must have cleaned up — .env must NOT exist on disk. # If F1 regressed (string-interpolated trap), the file would still be here. [ ! -f "$QUOTED_DIR/.env" ] @@ -842,27 +842,27 @@ PKG run "$SECRETS_BIN" which [ "$status" -eq 0 ] # The symlink should be ignored, falling through to default - [[ "$output" != *"/etc/passwd"* ]] - [[ "$output" == *"$HOME/.secrets"* ]] - [[ "$output" == *"source: default"* ]] + [[ "$output" != *"/etc/passwd"* ]] || false + [[ "$output" == *"$HOME/.secrets"* ]] || false + [[ "$output" == *"source: default"* ]] || false } @test "F3: --store rejects flag-shaped value" { run "$SECRETS_BIN" --store --workspaces which [ "$status" -eq 1 ] - [[ "$output" == *"looks like a flag"* ]] + [[ "$output" == *"looks like a flag"* ]] || false } @test "F3: --store rejects literal --" { run "$SECRETS_BIN" --store -- which [ "$status" -eq 1 ] - [[ "$output" == *"looks like a flag"* ]] + [[ "$output" == *"looks like a flag"* ]] || false } @test "F4: --store= empty value is rejected" { run "$SECRETS_BIN" --store= which [ "$status" -eq 1 ] - [[ "$output" == *"requires a value"* ]] + [[ "$output" == *"requires a value"* ]] || false } @test "F5: HOME unset gives directed error" { @@ -872,7 +872,7 @@ PKG run "$SECRETS_BIN" which export HOME="$SAVED_HOME" # restore before assertions in case bats relies on it [ "$status" -ne 0 ] - [[ "$output" == *"HOME"* ]] + [[ "$output" == *"HOME"* ]] || false } # ─── EGB-282: optional remote URL in .secrets-store ────────────────── @@ -886,7 +886,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "EGB-282: .secrets-store with URL parses both tokens" { @@ -898,9 +898,9 @@ PKG # that includes the actual URL (not the placeholder). run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/work-secrets.git $HOME/.secrets-work"* ]] + [[ "$output" == *"git clone git@github.com:acme/work-secrets.git $HOME/.secrets-work"* ]] || false # Placeholder must NOT appear when a real URL was supplied - [[ "$output" != *""* ]] + [[ "$output" != *""* ]] || false } @test "EGB-282: missing-store error still works without URL (placeholder)" { @@ -911,7 +911,7 @@ PKG run "$SECRETS_BIN" pull [ "$status" -eq 1 ] # No URL given — placeholder is the right behavior. - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282: https URL is preserved literally" { @@ -921,7 +921,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"https://github.com/acme/work-secrets.git"* ]] + [[ "$output" == *"https://github.com/acme/work-secrets.git"* ]] || false } @test "EGB-282: ~/-prefixed path with URL works" { @@ -931,7 +931,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/x.git $HOME/.secrets-x"* ]] + [[ "$output" == *"git clone git@github.com:acme/x.git $HOME/.secrets-x"* ]] || false } @test "EGB-282: comments before URL line are still skipped" { @@ -941,7 +941,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@github.com:acme/work-secrets.git"* ]] + [[ "$output" == *"git clone git@github.com:acme/work-secrets.git"* ]] || false } # ─── EGB-282 adversarial regressions: URL injection prevention ──────── @@ -958,11 +958,11 @@ PKG run "$SECRETS_BIN" pull [ "$status" -eq 1 ] # Must use the placeholder, NOT the attacker URL - [[ "$output" == *""* ]] - [[ "$output" != *"rm -rf"* ]] + [[ "$output" == *""* ]] || false + [[ "$output" != *"rm -rf"* ]] || false # And must have warned the user that something was dropped - [[ "$output" == *"WARNING"* ]] - [[ "$output" == *"unsafe"* ]] + [[ "$output" == *"WARNING"* ]] || false + [[ "$output" == *"unsafe"* ]] || false } @test "EGB-282 SECURITY: URL with backticks is dropped" { @@ -972,7 +972,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: URL with command substitution \$() is dropped" { @@ -982,7 +982,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: URL with ANSI escape is dropped (terminal-spoof prevention)" { @@ -993,7 +993,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: multi-token URL ('work url1 url2') is dropped" { @@ -1005,7 +1005,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282 SECURITY: glob char in URL is dropped (no expansion either way)" { @@ -1018,7 +1018,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *""* ]] + [[ "$output" == *""* ]] || false } @test "EGB-282: spec parsing is glob-safe (work * does NOT expand)" { @@ -1032,7 +1032,7 @@ PKG [ "$status" -eq 0 ] # Spec is the literal "work" (resolves to ~/.secrets-work). The "*" gets # rejected as unsafe URL and dropped. Resolution works; no globbing. - [[ "$output" == *"$HOME/.secrets-work"* ]] + [[ "$output" == *"$HOME/.secrets-work"* ]] || false } @test "EGB-282: URL with - + _ : / @ . is preserved (positive test)" { @@ -1043,7 +1043,7 @@ PKG cd "$WORK_DIR/proj" run "$SECRETS_BIN" pull [ "$status" -eq 1 ] - [[ "$output" == *"git+ssh://user@host:2222/path/to-repo_v2.git"* ]] + [[ "$output" == *"git+ssh://user@host:2222/path/to-repo_v2.git"* ]] || false } # ─── EGB-531: gradle.properties external file support ────────────────── @@ -1069,9 +1069,9 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"gradle-properties"* ]] - [[ "$output" == *"~/.gradle/gradle.properties"* ]] - [[ "$output" == *"beaconClerkPkTest"* ]] + [[ "$output" == *"gradle-properties"* ]] || false + [[ "$output" == *"~/.gradle/gradle.properties"* ]] || false + [[ "$output" == *"beaconClerkPkTest"* ]] || false } @test "EGB-531: push extracts managed keys into external/ blob (no .env needed)" { @@ -1080,7 +1080,7 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"Extracted 2 key"* ]] + [[ "$output" == *"Extracted 2 key"* ]] || false run bash -c "ls $SECRETS_DIR/gproj/external/*.gradle-properties.age" [ "$status" -eq 0 ] } @@ -1091,7 +1091,7 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"not found"* ]] + [[ "$output" == *"not found"* ]] || false } @test "EGB-531: pull merges managed keys, preserves unrelated entries" { @@ -1103,7 +1103,7 @@ gradle_project() { gradle_src $'unrelated.key=keepme\norg.gradle.jvmargs=-Xmx2g\n' run "$SECRETS_BIN" pull gproj [ "$status" -eq 0 ] - [[ "$output" == *"Merged 2 key"* ]] + [[ "$output" == *"Merged 2 key"* ]] || false grep -q 'beaconClerkPkTest=pk_test_abc' "$HOME/.gradle/gradle.properties" grep -q 'beaconClerkPkLive=pk_live_xyz' "$HOME/.gradle/gradle.properties" grep -q 'unrelated.key=keepme' "$HOME/.gradle/gradle.properties" @@ -1225,7 +1225,7 @@ gradle_project() { printf 'gradle-properties ~/.gradle/custom.properties beaconClerkPkTest\n' > .secrets-files run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"gradle.properties"* ]] + [[ "$output" == *"gradle.properties"* ]] || false } @test "EGB-531: target outside HOME is refused" { @@ -1239,7 +1239,7 @@ gradle_project() { run "$SECRETS_BIN" push gproj rm -rf "$outside" [ "$status" -eq 1 ] - [[ "$output" == *"HOME"* ]] + [[ "$output" == *"HOME"* ]] || false } @test "EGB-531: symlinked target is refused" { @@ -1250,7 +1250,7 @@ gradle_project() { gradle_project gproj beaconClerkPkTest run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"symlink"* ]] + [[ "$output" == *"symlink"* ]] || false } @test "EGB-531: unknown type in manifest warns and skips" { @@ -1259,7 +1259,7 @@ gradle_project() { printf 'gradle-props ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files echo "X=1" > .env run "$SECRETS_BIN" push gproj - [[ "$output" == *"unknown type"* ]] + [[ "$output" == *"unknown type"* ]] || false [ ! -d "$SECRETS_DIR/gproj/external" ] } @@ -1269,7 +1269,7 @@ gradle_project() { printf 'gradle-properties\n' > .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"WARNING"* ]] + [[ "$output" == *"WARNING"* ]] || false } @test "EGB-531: manifest path with command-substitution chars is rejected" { @@ -1280,7 +1280,7 @@ gradle_project() { printf 'gradle-properties ~/.gradle/gradle.properties$(touch %s) beaconClerkPkTest\n' "$pwn" > .secrets-files run "$SECRETS_BIN" which [ ! -f "$pwn" ] - [[ "$output" == *"WARNING"* ]] + [[ "$output" == *"WARNING"* ]] || false } @test "EGB-531: symlinked .secrets-files is ignored" { @@ -1290,7 +1290,7 @@ gradle_project() { ln -s "$HOME/realmanifest" .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" != *"beaconClerkPkTest"* ]] + [[ "$output" != *"beaconClerkPkTest"* ]] || false } @test "EGB-531: rekey re-encrypts the external blob (still decryptable after)" { @@ -1329,7 +1329,7 @@ gradle_project() { "$SECRETS_BIN" push gproj >/dev/null 2>&1 run "$SECRETS_BIN" list [ "$status" -eq 0 ] - [[ "$output" == *"external"* ]] + [[ "$output" == *"external"* ]] || false } @test "EGB-531: no .secrets-files behaves exactly as before (backward compat)" { @@ -1347,7 +1347,7 @@ gradle_project() { git add -f gradle.properties run git commit -m "should fail" [ "$status" -eq 1 ] - [[ "$output" == *"Plaintext"* ]] + [[ "$output" == *"Plaintext"* ]] || false } # ── EGB-531: coverage for warning/error branches, workspaces, multi-entry ── @@ -1395,9 +1395,9 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"beaconClerkPkLive"* ]] - [[ "$output" == *"not found"* ]] - [[ "$output" == *"Extracted 1 key"* ]] + [[ "$output" == *"beaconClerkPkLive"* ]] || false + [[ "$output" == *"not found"* ]] || false + [[ "$output" == *"Extracted 1 key"* ]] || false } @test "EGB-531: pull warns when manifest entry has no blob in store" { @@ -1408,7 +1408,7 @@ gradle_project() { cd "$WORK_DIR/gproj" run "$SECRETS_BIN" pull gproj [ "$status" -eq 0 ] - [[ "$output" == *"no encrypted data exists"* ]] + [[ "$output" == *"no encrypted data exists"* ]] || false } @test "EGB-531: multi-entry manifest syncs each target" { @@ -1436,8 +1436,8 @@ gradle_project() { printf 'gradle-properties ~/.gradle/gradle.properties bad=key\n' > .secrets-files run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"WARNING"* ]] - [[ "$output" != *"bad=key"* ]] + [[ "$output" == *"WARNING"* ]] || false + [[ "$output" != *"bad=key"* ]] || false } @test "EGB-531: symlinked parent dir of target is refused" { @@ -1448,7 +1448,7 @@ gradle_project() { gradle_project gproj beaconClerkPkTest run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"symlink"* ]] + [[ "$output" == *"symlink"* ]] || false } @test "EGB-531: push skips a multi-line (continuation) managed value with a warning" { @@ -1459,8 +1459,8 @@ gradle_project() { gradle_project gproj run "$SECRETS_BIN" push gproj [ "$status" -eq 0 ] - [[ "$output" == *"multi-line"* ]] - [[ "$output" == *"Extracted 1 key"* ]] + [[ "$output" == *"multi-line"* ]] || false + [[ "$output" == *"Extracted 1 key"* ]] || false } @test "EGB-531: push skips comment and continuation lines in source" { @@ -1491,7 +1491,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone"* ]] + [[ "$output" == *"git clone"* ]] || false # Must not leave a half-initialized store behind [ ! -d "$SECRETS_DIR/.git" ] # Key untouched @@ -1505,12 +1505,12 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"Restored store .gitignore"* ]] + [[ "$output" == *"Restored store .gitignore"* ]] || false [ -f "$SECRETS_DIR/.gitignore" ] grep -q "key.txt" "$SECRETS_DIR/.gitignore" # key.txt must never be tracked (push does `git add -A` in the store) run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "pull restores missing store .gitignore" { @@ -1535,7 +1535,7 @@ gradle_project() { [ "$status" -eq 0 ] [ -f "$SECRETS_DIR/.gitignore" ] run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "rekey re-encrypts dotenv blobs (round-trip survives key rotation)" { @@ -1563,7 +1563,7 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"Reinstalled pre-commit hook"* ]] + [[ "$output" == *"Reinstalled pre-commit hook"* ]] || false [ -x "$SECRETS_DIR/.git/hooks/pre-commit" ] } @@ -1584,8 +1584,8 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" != *"Restored store .gitignore"* ]] - [[ "$output" != *"Reinstalled pre-commit hook"* ]] + [[ "$output" != *"Restored store .gitignore"* ]] || false + [[ "$output" != *"Reinstalled pre-commit hook"* ]] || false } @test "restored store .gitignore carries the full block/allow globs" { @@ -1617,7 +1617,7 @@ gradle_project() { [ "$status" -eq 0 ] [ -f "$SECRETS_DIR/.gitignore" ] run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "push untracks a previously committed key.txt with a warning" { @@ -1629,9 +1629,9 @@ gradle_project() { run "$SECRETS_BIN" push [ "$status" -eq 0 ] - [[ "$output" == *"key.txt was tracked"* ]] + [[ "$output" == *"key.txt was tracked"* ]] || false run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "push rewrites a store .gitignore that is missing the key.txt line" { @@ -1643,7 +1643,7 @@ gradle_project() { [ "$status" -eq 0 ] grep -qx 'key.txt' "$SECRETS_DIR/.gitignore" run git -C "$SECRETS_DIR" ls-files - [[ "$output" != *"key.txt"* ]] + [[ "$output" != *"key.txt"* ]] || false } @test "init guard renders the real clone URL when .secrets-store carries a remote" { @@ -1655,7 +1655,7 @@ gradle_project() { run "$SECRETS_BIN" init [ "$status" -eq 1 ] - [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] + [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] || false } # ─── EGB-652: `file` external type (whole-file sync, e.g. Android keystore) ── @@ -1680,8 +1680,8 @@ file_project() { file_project fproj run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"file"* ]] - [[ "$output" == *"~/keystores/upload.keystore"* ]] + [[ "$output" == *"file"* ]] || false + [[ "$output" == *"~/keystores/upload.keystore"* ]] || false } @test "EGB-652: push encrypts a file-type entry into external/ blob" { @@ -1690,7 +1690,7 @@ file_project() { file_project fproj run "$SECRETS_BIN" push fproj [ "$status" -eq 0 ] - [[ "$output" == *"Encrypted file"* ]] + [[ "$output" == *"Encrypted file"* ]] || false run bash -c "ls $SECRETS_DIR/fproj/external/*.file.age" [ "$status" -eq 0 ] } @@ -1704,7 +1704,7 @@ file_project() { rm -rf "$HOME/keystores" run "$SECRETS_BIN" pull fproj [ "$status" -eq 0 ] - [[ "$output" == *"Restored file"* ]] + [[ "$output" == *"Restored file"* ]] || false cmp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/reference" mode=$(stat -f '%Lp' "$HOME/keystores/upload.keystore" 2>/dev/null || stat -c '%a' "$HOME/keystores/upload.keystore") [ "$mode" = "600" ] @@ -1730,10 +1730,10 @@ file_project() { cd "$dir" run "$SECRETS_BIN" which [ "$status" -eq 0 ] - [[ "$output" == *"take no keys"* ]] + [[ "$output" == *"take no keys"* ]] || false # The rejected entry must not be listed as parsed (header only prints # when at least one entry parses). - [[ "$output" != *"external files ("* ]] + [[ "$output" != *"external files ("* ]] || false } @test "EGB-652: file target outside HOME is refused on push" { @@ -1743,7 +1743,7 @@ file_project() { cd "$dir" run "$SECRETS_BIN" push fout [ "$status" -ne 0 ] - [[ "$output" == *"inside \$HOME"* ]] || [[ "$output" == *"Refusing"* ]] + [[ "$output" == *"inside \$HOME"* ]] || [[ "$output" == *"Refusing"* ]] || false } @test "EGB-652: gradle-properties entries still work alongside a file entry" { @@ -1755,6 +1755,6 @@ file_project() { cd "$dir" run "$SECRETS_BIN" push fmix [ "$status" -eq 0 ] - [[ "$output" == *"Extracted 1 key"* ]] - [[ "$output" == *"Encrypted file"* ]] + [[ "$output" == *"Extracted 1 key"* ]] || false + [[ "$output" == *"Encrypted file"* ]] || false } From 18018dbd3b4e03e46e8af67b8ddc6ee373a4ddc6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:24:35 -0700 Subject: [PATCH 02/12] =?UTF-8?q?feat:=20.secrets.json=20manifest=20core?= =?UTF-8?q?=20=E2=80=94=20add=20command,=20rails,=20canonical=20form=20(EG?= =?UTF-8?q?B-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - secrets add : explicit manifest writer, idempotent, atomic write - _validate_dotenv_rel_path: project-relative confinement rail (no .. / absolute / shell metas; @ allowed for npm-scoped workspace dirs) - _check_manifest_file: refuses symlinks, malformed JSON (jq error with file named), unsupported schema versions (directed upgrade error) - canonical serialization: jq --sort-keys + sorted/deduped dotenv — add order produces byte-identical manifests - which: validates + summarizes the manifest (doubles as linter) - jq required only when a manifest exists/is written --- secrets | 147 +++++++++++++++++++++++++++++++++++++++++++++ test/manifest.bats | 125 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 test/manifest.bats diff --git a/secrets b/secrets index 508e5de..99fdae4 100755 --- a/secrets +++ b/secrets @@ -712,6 +712,129 @@ pull_external_files() { done < <(_parse_secrets_files_manifest "$manifest") } +# ─── Manifest (.secrets.json) — EGB-677 store format v2, stage 1 ────── +# +# A committed, project-root JSON manifest declaring everything the project +# syncs. Stage 1 is ADDITIVE: the manifest is read alongside the existing +# dotenv globs and `.secrets-files`; the v1 store layout is unchanged. +# +# { +# "version": 2, +# "options": { "autoAdd": true }, +# "dotenv": [".env", "packages/web/.env.development"], +# "external": [ +# { "type": "properties", "path": "~/.gradle/gradle.properties", +# "keys": ["beaconClerkPk"] }, +# { "type": "file", "path": "~/keystores/beacon-upload.keystore" } +# ] +# } +# +# Security posture mirrors `.secrets-store`/`.secrets-files`: symlinked +# manifests are refused, dotenv paths are confined to the project root +# (no `..`, no absolute paths, conservative charset — `@` allowed for +# npm-scoped workspace dirs), and every jq-extracted string is +# re-validated before any filesystem use. jq is required only when a +# manifest exists or is being written — manifest-less projects keep +# working with zero new dependencies. + +SECRETS_JSON_NAME=".secrets.json" +MANIFEST_VERSION=2 + +# Validate a project-relative dotenv path from the manifest (or `secrets +# add`). The path is attacker-influenced in team repos (committed file), +# so this is a security rail, not just hygiene. Prints an error and +# returns 1 when unsafe. +_validate_dotenv_rel_path() { + local p="$1" + if [ -z "$p" ]; then + echo "ERROR: $SECRETS_JSON_NAME: empty dotenv path." >&2 + return 1 + fi + case "$p" in + /*) + echo "ERROR: $SECRETS_JSON_NAME: '$p' is absolute — dotenv paths must be project-relative." >&2 + return 1 + ;; + *..*) + echo "ERROR: $SECRETS_JSON_NAME: '$p' contains '..' — dotenv paths must be project-relative (no traversal)." >&2 + return 1 + ;; + *[!A-Za-z0-9@/._-]*) + echo "ERROR: $SECRETS_JSON_NAME: unsafe characters in '$p'. Paths may contain letters, digits, @ / . _ - only." >&2 + return 1 + ;; + esac + return 0 +} + +# Validate the manifest file itself (existence assumed checked by caller): +# refuse symlinks, malformed JSON, and unsupported schema versions — each +# with a directed error. $1 = manifest path. +_check_manifest_file() { + local manifest="$1" + if [ -L "$manifest" ]; then + die "$manifest is a symlink. Refusing to read it. + A committed symlink could point the manifest at attacker-chosen content. + Replace it with a regular file." + fi + check_cmd jq + local jq_err + if ! jq_err=$(jq -e . "$manifest" 2>&1 >/dev/null); then + die "$manifest: invalid JSON. + $jq_err + Fix the syntax (or delete the file and re-run 'secrets add' / 'secrets push')." + fi + local ver + ver=$(jq -r '.version // "missing"' "$manifest") + if [ "$ver" != "$MANIFEST_VERSION" ]; then + die "$manifest: manifest version $ver is not supported. + This client supports version $MANIFEST_VERSION. + If the manifest was written by a newer secrets, upgrade this machine: + git -C $SCRIPT_DIR pull" + fi +} + +# Canonically (re)write the manifest: sorted keys, sorted+deduped dotenv, +# atomic tmp+mv in the project dir. stdin = the new JSON document. +_write_manifest_canonical() { + local manifest="$1" + local dir; dir=$(dirname "$manifest") + local tmp; tmp=$(mktemp "$dir/.secrets-json.XXXXXX") || return 1 + if ! jq --sort-keys '.dotenv |= ((. // []) | unique | sort)' > "$tmp"; then + rm -f "$tmp" + return 1 + fi + mv "$tmp" "$manifest" +} + +# `secrets add ` — the explicit manifest writer. Creates +# .secrets.json on first use; validates and dedupes thereafter. +cmd_add() { + local p="${1:-}" + [ -n "$p" ] || die "Usage: secrets add " + # Normalize a leading ./ + p="${p#./}" + _validate_dotenv_rel_path "$p" || exit 1 + if [ ! -f "$PWD/$p" ]; then + die "'$p' not found in $PWD. + Create the file first, then re-run: secrets add $p" + fi + check_cmd jq + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + jq --arg p "$p" '.dotenv = ((.dotenv // []) + [$p])' "$manifest" \ + | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" + else + jq -n --arg p "$p" '{version: '"$MANIFEST_VERSION"', dotenv: [$p]}' \ + | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" + fi + info "Added '$p' to $SECRETS_JSON_NAME" + info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." +} + +# ─── End manifest ────────────────────────────────────────────────────── + # Read package.json workspaces and expand globs to actual directories. # Prints one workspace path per line (relative to the monorepo root). get_workspaces() { @@ -1397,6 +1520,29 @@ cmd_which() { echo "store: $SECRETS_DIR" echo "source: $STORE_SOURCE" + # v2 manifest (.secrets.json): validate and summarize. Validation here + # is deliberately fatal (symlink / malformed / unsupported version) so + # `secrets which` doubles as the manifest linter. + local json_manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$json_manifest" ]; then + _check_manifest_file "$json_manifest" + echo "manifest ($SECRETS_JSON_NAME at $json_manifest):" + local entry + while IFS= read -r entry; do + [ -n "$entry" ] || continue + if _validate_dotenv_rel_path "$entry"; then + echo " dotenv $entry" + else + echo " dotenv $entry [UNSAFE — will be refused]" + fi + done < <(jq -r '.dotenv // [] | .[]' "$json_manifest") + local etype epath ekeys + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + echo " $etype $epath $ekeys" + done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$json_manifest") + fi + # Read back any external-file manifest in cwd (validates the format and # gives the user a way to confirm it parsed, since there's no add-file # command). Skips symlinked manifests. @@ -1569,6 +1715,7 @@ case "${1:-help}" in shift cmd_run "$@" ;; + add) cmd_add "${2:-}" ;; list) cmd_list ;; rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; diff --git a/test/manifest.bats b/test/manifest.bats new file mode 100644 index 0000000..eb3e7d2 --- /dev/null +++ b/test/manifest.bats @@ -0,0 +1,125 @@ +#!/usr/bin/env bats +# EGB-677 stage 1: .secrets.json manifest — parse, rails, add, generators. + +load test_helper + +# ─── A: manifest core — secrets add + rails + canonical form ────────── + +@test "add creates .secrets.json with version 2 and the dotenv entry" { + create_project_dir addproj + run "$SECRETS_BIN" add .env + [ "$status" -eq 0 ] + [ -f ".secrets.json" ] + run jq -r '.version' .secrets.json + [ "$output" = "2" ] + run jq -r '.dotenv[0]' .secrets.json + [ "$output" = ".env" ] +} + +@test "add is idempotent — no duplicate entries" { + create_project_dir addproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" add .env + [ "$status" -eq 0 ] + run jq -r '.dotenv | length' .secrets.json + [ "$output" = "1" ] +} + +@test "add accepts nested workspace paths" { + create_project_dir addproj + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + run "$SECRETS_BIN" add packages/web/.env.development + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/web/.env.development") != null' .secrets.json + [ "$output" = "true" ] +} + +@test "add accepts npm-scoped workspace paths (@)" { + create_project_dir addproj + mkdir -p "packages/@acme/web" + echo "K=v" > "packages/@acme/web/.env" + run "$SECRETS_BIN" add "packages/@acme/web/.env" + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/@acme/web/.env") != null' .secrets.json + [ "$output" = "true" ] +} + +@test "add rejects path traversal (..)" { + create_project_dir addproj + run "$SECRETS_BIN" add ../escape/.env + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false + [ ! -f ".secrets.json" ] +} + +@test "add rejects absolute paths" { + create_project_dir addproj + run "$SECRETS_BIN" add /etc/passwd + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false + [ ! -f ".secrets.json" ] +} + +@test "add rejects shell metacharacters in path" { + create_project_dir addproj + run "$SECRETS_BIN" add '.env;rm -rf ~' + [ "$status" -eq 1 ] + [ ! -f ".secrets.json" ] +} + +@test "add requires the file to exist" { + create_project_dir addproj + run "$SECRETS_BIN" add .env.missing + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] || false +} + +@test "manifest serialization is canonical — order of adds does not matter" { + create_project_dir addproj + echo "A=1" > .env.alpha + echo "B=2" > .env.beta + "$SECRETS_BIN" add .env.alpha >/dev/null + "$SECRETS_BIN" add .env.beta >/dev/null + cp .secrets.json "$TEST_TMPDIR/order1.json" + rm .secrets.json + "$SECRETS_BIN" add .env.beta >/dev/null + "$SECRETS_BIN" add .env.alpha >/dev/null + cmp -s .secrets.json "$TEST_TMPDIR/order1.json" +} + +@test "which shows manifest summary when .secrets.json is present" { + create_project_dir addproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *".env"* ]] || false +} + +@test "malformed .secrets.json dies with a directed error naming the file" { + create_project_dir addproj + echo '{ not json' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *"invalid"* ]] || false +} + +@test "unsupported manifest version dies with a directed upgrade error" { + create_project_dir addproj + echo '{"version": 99, "dotenv": [".env"]}' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *"version 99"* ]] || false + [[ "$output" == *"supports"* ]] || false +} + +@test "symlinked .secrets.json is refused" { + create_project_dir addproj + echo '{"version":2,"dotenv":[".env"]}' > "$TEST_TMPDIR/real-manifest.json" + ln -s "$TEST_TMPDIR/real-manifest.json" .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 1 ] + [[ "$output" == *"symlink"* ]] || false +} From 884da0965cfebc99bd203c52ab4a268b1747a0fa Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:32:17 -0700 Subject: [PATCH 03/12] =?UTF-8?q?feat:=20manifest-aware=20push=20=E2=80=94?= =?UTF-8?q?=20generator=20auto-add,=20autoAdd=20toggle,=20--frozen/--dry-r?= =?UTF-8?q?un=20(EGB-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - push syncs FROM the manifest; v1 store layout unchanged (nested entries land at /.age, same shape -w always used) - discovery (root globs + quiet package.json workspace re-scan when a manifest exists) feeds the manifest as a generator; new files auto-add with ==> notice + undo guidance - options.autoAdd committed toggle (default ON when absent); explicit false warns on undeclared files instead of enrolling them - push --frozen: declared-only for one invocation; push --dry-run: reports would-add/would-sync, touches nothing - bootstrap ordering: manifest written only after >=1 blob encrypts - declared-but-missing warns and continues; unsafe manifest path dies - jq // falsy gotcha: explicit autoAdd:false compared directly --- secrets | 147 ++++++++++++++++++++++++++++++++++++++++++++- test/manifest.bats | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 3 deletions(-) diff --git a/secrets b/secrets index 99fdae4..e1d4f2c 100755 --- a/secrets +++ b/secrets @@ -833,6 +833,27 @@ cmd_add() { info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." } +# Quietly emit "ws-dir/basename" for every env file in a package.json +# workspace under . Emits nothing (and never dies) when is +# not a workspace monorepo or jq is unavailable — plain `push` calls this +# speculatively so a new workspace's env files keep getting discovered +# after the one-time --workspaces generator run (EGB-677 E13). +_maybe_workspace_env_files() { + local root="$1" + [ -f "$root/package.json" ] || return 0 + command -v jq >/dev/null 2>&1 || return 0 + jq -e '.workspaces' "$root/package.json" >/dev/null 2>&1 || return 0 + local ws f + while IFS= read -r ws; do + [ -n "$ws" ] || continue + if collect_env_files "$root/$ws"; then + for f in "${COLLECTED_FILES[@]}"; do + printf '%s/%s\n' "$ws" "$(basename "$f")" + done + fi + done < <(get_workspaces "$root") +} + # ─── End manifest ────────────────────────────────────────────────────── # Read package.json workspaces and expand globs to actual directories. @@ -1033,7 +1054,22 @@ commit_and_push_secrets() { fi } +# Manifest-aware push (EGB-677 stage 1). Discovery (root globs + a quiet +# package.json workspace re-scan when a manifest exists) feeds the +# manifest as a GENERATOR; the sync itself runs FROM the manifest. The +# v1 store layout is unchanged: root files land at /.age, +# nested entries at /.age (same shape -w always used). cmd_push() { + local frozen=false dry_run=false explicit_project="" + while [ $# -gt 0 ]; do + case "$1" in + --frozen) frozen=true; shift ;; + --dry-run) dry_run=true; shift ;; + -*) die "Unknown push flag: $1. Usage: secrets push [--frozen] [--dry-run] [project]" ;; + *) explicit_project="$1"; shift ;; + esac + done + check_cmd age check_cmd git resolve_store @@ -1041,20 +1077,124 @@ cmd_push() { check_key local project - project=$(derive_project_name "${1:-}") + project=$(derive_project_name "$explicit_project") info "Pushing secrets for project: $project" echo_store_if_non_default local pubkey pubkey=$(get_pubkey) + # ── Manifest read (validated; absence = bootstrap) ── + local manifest="$PWD/$SECRETS_JSON_NAME" + local have_manifest=false auto_add=true declared="" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + have_manifest=true + declared=$(jq -r '.dotenv // [] | .[]' "$manifest") + local d + while IFS= read -r d; do + [ -n "$d" ] || continue + _validate_dotenv_rel_path "$d" \ + || die "Refusing unsafe dotenv path in $SECRETS_JSON_NAME (paths must be project-relative): $d" + done <<< "$declared" + # NB: jq's // treats false as empty, so `.options.autoAdd // true` + # would silently flip an explicit false back to true. Compare directly. + auto_add=$(jq -r '.options.autoAdd | if . == false then "false" else "true" end' "$manifest") + fi + [ "$frozen" = true ] && auto_add=false + + # ── Discovery: root globs + workspace re-scan (manifest projects) ── + local discovered="" f + if collect_env_files "$PWD"; then + for f in "${COLLECTED_FILES[@]}"; do + discovered="$discovered$(basename "$f")"$'\n' + done + fi + if [ "$have_manifest" = true ]; then + discovered="$discovered$(_maybe_workspace_env_files "$PWD")"$'\n' + fi + + # to_add = discovered − declared (deduped; pure bash 3.2, no assoc arrays) + local to_add="" e known + while IFS= read -r e; do + [ -n "$e" ] || continue + known=0 + while IFS= read -r d; do [ "$d" = "$e" ] && { known=1; break; }; done <<< "$declared" + [ "$known" -eq 1 ] && continue + while IFS= read -r d; do [ "$d" = "$e" ] && { known=1; break; }; done <<< "$to_add" + [ "$known" -eq 1 ] && continue + to_add="$to_add$e"$'\n' + done <<< "$discovered" + + if [ "$dry_run" = true ]; then + info "Dry run — nothing encrypted, nothing written." + if [ -n "$to_add" ]; then + echo "Would add to $SECRETS_JSON_NAME:" + while IFS= read -r e; do [ -n "$e" ] && echo " $e"; done <<< "$to_add" + else + echo "Nothing new to add to $SECRETS_JSON_NAME." + fi + if [ -n "$declared" ]; then + echo "Would sync (declared):" + while IFS= read -r e; do [ -n "$e" ] && echo " $e"; done <<< "$declared" + fi + return 0 + fi + + # ── Build the sync list ── + local sync_list="$declared" + if [ "$auto_add" = true ] || [ "$have_manifest" = false ]; then + sync_list="$declared"$'\n'"$to_add" + else + while IFS= read -r e; do + [ -n "$e" ] || continue + echo "WARNING: '$e' is not declared in $SECRETS_JSON_NAME and autoAdd is off — not synced. Run: secrets add $e" >&2 + done <<< "$to_add" + fi + + # ── Encrypt FROM the (effective) manifest ── + local count=0 rel + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if [ ! -f "$PWD/$rel" ]; then + echo "WARNING: '$rel' is declared in $SECRETS_JSON_NAME but not found in $PWD — skipping." >&2 + continue + fi + case "$rel" in + */*) mkdir -p "$SECRETS_DIR/$project/$(dirname "$rel")" ;; + *) mkdir -p "$SECRETS_DIR/$project" ;; + esac + age -r "$pubkey" -o "$SECRETS_DIR/$project/${rel}.age" "$PWD/$rel" + echo " $rel" + count=$((count + 1)) + done <<< "$sync_list" + [ "$count" -gt 0 ] && info "$project: $count file(s)" + local did=0 - if push_dir_to_project "$PWD" "$project" "$pubkey"; then did=1; fi + [ "$count" -gt 0 ] && did=1 if push_external_files "$PWD" "$project" "$pubkey"; then did=1; fi if [ "$did" -eq 0 ]; then die "No secret files (.env, .env.*, .dev.vars) or $SECRETS_FILES_NAME entries found in $PWD" fi + # ── Manifest write AFTER successful encryption (bootstrap ordering) ── + if [ "$count" -gt 0 ] && [ -n "$to_add" ] && [ "$frozen" = false ] \ + && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then + local add_json + add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') + if [ "$have_manifest" = true ]; then + jq --argjson add "$add_json" '.dotenv = ((.dotenv // []) + $add)' "$manifest" \ + | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" + else + jq -n --argjson add "$add_json" '{version: '"$MANIFEST_VERSION"', dotenv: $add}' \ + | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" + fi + while IFS= read -r e; do + [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" + done <<< "$to_add" + info "Commit the manifest so other machines pick it up. To undo an entry: edit $SECRETS_JSON_NAME (or use 'secrets push --frozen' to skip auto-add)." + fi + commit_and_push_secrets "update $project" } @@ -1694,7 +1834,8 @@ case "${1:-help}" in if [ "${2:-}" = "-w" ] || [ "${2:-}" = "--workspaces" ]; then cmd_push_workspaces else - cmd_push "${2:-}" + shift + cmd_push "$@" fi ;; pull) diff --git a/test/manifest.bats b/test/manifest.bats index eb3e7d2..9800c72 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -123,3 +123,124 @@ load test_helper [ "$status" -eq 1 ] [[ "$output" == *"symlink"* ]] || false } + +# ─── B: push from manifest — generators, autoAdd, --frozen/--dry-run ─── + +@test "push with manifest syncs nested declared file into v1 store layout" { + init_with_remote + create_project_dir nestproj + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/nestproj/packages/web/.env.development.age" ] +} + +@test "push auto-adds newly discovered root files to an existing manifest" { + init_with_remote + create_project_dir autoproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"Added"* ]] || false + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "true" ] + [ -f "$SECRETS_DIR/autoproj/.env.staging.age" ] +} + +@test "bootstrap: plain push creates the manifest from discovered files" { + init_with_remote + create_project_dir bootproj + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f ".secrets.json" ] + run jq -r '.dotenv | length' .secrets.json + [ "$output" = "2" ] +} + +@test "failed push leaves no bootstrap manifest behind" { + init_with_remote + mkdir -p "$WORK_DIR/emptyproj" + cd "$WORK_DIR/emptyproj" + run "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [ ! -f ".secrets.json" ] +} + +@test "autoAdd=false: undeclared discovered file is warned about, not added or synced" { + init_with_remote + create_project_dir noaddproj + printf '{"version":2,"options":{"autoAdd":false},"dotenv":[".env"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"not declared"* ]] || false + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "false" ] + [ -f "$SECRETS_DIR/noaddproj/.env.age" ] + [ ! -f "$SECRETS_DIR/noaddproj/.env.staging.age" ] +} + +@test "push --frozen skips auto-add even when autoAdd is on" { + init_with_remote + create_project_dir frozenproj + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push --frozen + [ "$status" -eq 0 ] + run jq -r '.dotenv | index(".env.staging") != null' .secrets.json + [ "$output" = "false" ] + [ ! -f "$SECRETS_DIR/frozenproj/.env.staging.age" ] + # declared entry still synced under the REAL project name + [ -f "$SECRETS_DIR/frozenproj/.env.age" ] +} + +@test "push --dry-run reports would-add entries and changes nothing" { + init_with_remote + create_project_dir dryproj + "$SECRETS_BIN" add .env >/dev/null + cp .secrets.json "$TEST_TMPDIR/manifest-before.json" + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *".env.staging"* ]] || false + cmp -s .secrets.json "$TEST_TMPDIR/manifest-before.json" + [ ! -f "$SECRETS_DIR/dryproj/.env.age" ] + # nothing committed to the store at all + [ "$(git -C "$SECRETS_DIR" rev-list --count HEAD)" -eq 1 ] +} + +@test "plain push re-scans package.json workspaces when a manifest exists" { + init_with_remote + local mono="$WORK_DIR/wsproj" + mkdir -p "$mono/packages/api" + printf '{"workspaces": ["packages/*"]}\n' > "$mono/package.json" + echo "ROOT=1" > "$mono/.env" + echo "API=1" > "$mono/packages/api/.dev.vars" + git init "$mono" >/dev/null 2>&1 + cd "$mono" + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + run jq -r '.dotenv | index("packages/api/.dev.vars") != null' .secrets.json + [ "$output" = "true" ] + [ -f "$SECRETS_DIR/wsproj/packages/api/.dev.vars.age" ] +} + +@test "declared-but-missing file warns and push continues" { + init_with_remote + create_project_dir missproj + "$SECRETS_BIN" add .env >/dev/null + printf '{"version":2,"dotenv":[".env",".env.gone"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *".env.gone"* ]] || false + [ -f "$SECRETS_DIR/missproj/.env.age" ] +} + +@test "unsafe dotenv entry in a committed manifest dies on push" { + init_with_remote + create_project_dir evilproj + printf '{"version":2,"dotenv":["../escape/.env"]}\n' > .secrets.json + run "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [[ "$output" == *"project-relative"* ]] || false +} From d7e1400487eacb9b41120a9b596450e688a41c11 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:44:02 -0700 Subject: [PATCH 04/12] feat: external entries via .secrets.json + legacy absorb + properties rail (EGB-677 stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .secrets.json external[] drives push/pull: type 'properties' (alias of gradle-properties; blob suffix stays legacy-compatible in stage 1) and type 'file'; same charset rails as the legacy parser - push absorbs uncovered .secrets-files entries into the manifest (idempotent, gradle-properties → properties) with a delete hint - pull: manifest wins entirely; a coexisting .secrets-files warns as superseded instead of being silently ignored - basename rail generalized: properties targets must end '.properties' (was exact 'gradle.properties') — rc files/gitconfig still blocked; EGB-531 wrong-basename test updated for the sanctioned change --- secrets | 206 +++++++++++++++++++++++++++++++++++++++------ test/manifest.bats | 135 +++++++++++++++++++++++++++++ test/secrets.bats | 13 ++- 3 files changed, 322 insertions(+), 32 deletions(-) diff --git a/secrets b/secrets index e1d4f2c..e02e636 100755 --- a/secrets +++ b/secrets @@ -425,9 +425,17 @@ _parse_secrets_files_manifest() { _validate_external_target_path() { local p="$1" mtype="${2:-gradle-properties}" local base; base=$(basename "$p") - if [ "$mtype" = "gradle-properties" ] && [ "$base" != "gradle.properties" ]; then - echo "ERROR: $SECRETS_FILES_NAME: target basename must be 'gradle.properties' (got '$base'). Refusing." >&2 - return 1 + if [ "$mtype" = "gradle-properties" ]; then + # EGB-677: generalized from exact 'gradle.properties' to any + # '*.properties' basename — still blocks merging key=value lines + # into ~/.bashrc / ~/.gitconfig style targets. + case "$base" in + *.properties) ;; + *) + echo "ERROR: properties target basename must end in '.properties' (got '$base'). Refusing." >&2 + return 1 + ;; + esac fi case "$p" in *..*) echo "ERROR: $SECRETS_FILES_NAME: target path may not contain '..'. Refusing." >&2; return 1 ;; esac local home_real; home_real=$(cd -P "$HOME" 2>/dev/null && pwd -P) || home_real="$HOME" @@ -575,13 +583,11 @@ merge_gradle_keys() { # there is no usable manifest. Dies on unsafe targets or all-missing keys. push_external_files() { local root="$1" project="$2" pubkey="$3" - local manifest="$root/$SECRETS_FILES_NAME" - [ -e "$manifest" ] || return 1 - if [ -L "$manifest" ]; then - echo "WARNING: $manifest is a symlink; ignoring." >&2 - return 1 - fi - [ -f "$manifest" ] || return 1 + # Entries come from .secrets.json (EGB-677) plus any legacy + # .secrets-files entries the manifest doesn't cover yet. + local entries + entries=$(_external_entries_for_push "$root") + [ -n "$entries" ] || return 1 local pushed=0 mtype mpath mkeys while IFS=$'\t' read -r mtype mpath mkeys; do @@ -631,7 +637,7 @@ push_external_files() { rm -f "$tmp" info "Extracted $found key(s) from $mpath" pushed=$((pushed + 1)) - done < <(_parse_secrets_files_manifest "$manifest") + done <<< "$entries" [ "$pushed" -gt 0 ] } @@ -641,13 +647,11 @@ push_external_files() { # missing blobs rather than aborting the whole pull. pull_external_files() { local root="$1" project="$2" - local manifest="$root/$SECRETS_FILES_NAME" - [ -e "$manifest" ] || return 0 - if [ -L "$manifest" ]; then - echo "WARNING: $manifest is a symlink; ignoring." >&2 - return 0 - fi - [ -f "$manifest" ] || return 0 + # .secrets.json wins entirely when present (EGB-677); legacy + # .secrets-files only drives manifest-less projects. + local entries + entries=$(_external_entries_for_pull "$root") + [ -n "$entries" ] || return 0 local mtype mpath mkeys while IFS=$'\t' read -r mtype mpath mkeys; do @@ -709,7 +713,7 @@ pull_external_files() { echo "WARNING: failed to merge keys into $expanded — target left unchanged." >&2 fi rm -f "$tmp" - done < <(_parse_secrets_files_manifest "$manifest") + done <<< "$entries" } # ─── Manifest (.secrets.json) — EGB-677 store format v2, stage 1 ────── @@ -833,6 +837,133 @@ cmd_add() { info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry." } +# Emit "\t\t" tuples from a .secrets.json external[] +# array — the same wire format _parse_secrets_files_manifest produces, so +# push_external_files / pull_external_files consume either source +# unchanged. JSON type 'properties' maps to the legacy tuple token +# 'gradle-properties' so blob suffixes (and existing store blobs) stay +# stable in stage 1. Applies the same conservative charset checks as the +# legacy parser — jq guarantees well-formed JSON, not safe VALUES. +_json_external_entries() { + local manifest="$1" + check_cmd jq + local etype epath ekeys + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + case "$etype" in + properties|gradle-properties) + etype="gradle-properties" + if [ -z "$ekeys" ]; then + echo "WARNING: $SECRETS_JSON_NAME: properties entry '$epath' has no keys. Skipping." >&2 + continue + fi + ;; + file) + if [ -n "$ekeys" ]; then + echo "WARNING: $SECRETS_JSON_NAME: 'file' entries take no keys ('$epath' lists '$ekeys'). Skipping." >&2 + continue + fi + ;; + *) + echo "WARNING: $SECRETS_JSON_NAME: unknown external type '$etype' (supported: properties file). Skipping." >&2 + continue + ;; + esac + case "$epath" in + ''|*[!A-Za-z0-9/._~-]*|*..*) + echo "WARNING: $SECRETS_JSON_NAME: unsafe characters in external path '$epath'. Skipping." >&2 + continue + ;; + esac + case "$ekeys" in + *[!A-Za-z0-9._\ -]*) + echo "WARNING: $SECRETS_JSON_NAME: unsafe characters in key list for '$epath'. Skipping." >&2 + continue + ;; + esac + printf '%s\t%s\t%s\n' "$etype" "$epath" "$ekeys" + done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$manifest") +} + +# External tuples for PUSH: .secrets.json entries first, then legacy +# .secrets-files entries whose (type, path) the manifest doesn't cover — +# the absorb set, which cmd_push folds into the manifest after a +# successful push so the two sources converge. +_external_entries_for_push() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + local seen="" t p k + if [ -f "$json" ] && [ ! -L "$json" ]; then + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + printf '%s\t%s\t%s\n' "$t" "$p" "$k" + seen="$seen$t|$p"$'\n' + done < <(_json_external_entries "$json") + fi + if [ -e "$legacy" ]; then + if [ -L "$legacy" ]; then + echo "WARNING: $legacy is a symlink; ignoring." >&2 + elif [ -f "$legacy" ]; then + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + case "$seen" in *"$t|$p"$'\n'*) continue ;; esac + printf '%s\t%s\t%s\n' "$t" "$p" "$k" + done < <(_parse_secrets_files_manifest "$legacy") + fi + fi +} + +# External tuples for PULL: the manifest wins entirely when present; +# legacy .secrets-files is only consulted in manifest-less projects. +_external_entries_for_pull() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + if [ -f "$json" ] && [ ! -L "$json" ]; then + if [ -f "$legacy" ] && [ ! -L "$legacy" ]; then + echo "WARNING: $legacy is superseded by $SECRETS_JSON_NAME and was ignored on pull. Run 'secrets push' to absorb it, then delete it." >&2 + fi + _json_external_entries "$json" + return 0 + fi + [ -e "$legacy" ] || return 0 + if [ -L "$legacy" ]; then + echo "WARNING: $legacy is a symlink; ignoring." >&2 + return 0 + fi + [ -f "$legacy" ] && _parse_secrets_files_manifest "$legacy" + return 0 +} + +# JSON array of legacy .secrets-files entries NOT yet in the manifest — +# what cmd_push absorbs. gradle-properties becomes 'properties' on the +# JSON side. Parser warnings suppressed (push_external_files re-parses +# and warns once). +_legacy_absorb_json() { + local root="$1" + local json="$root/$SECRETS_JSON_NAME" legacy="$root/$SECRETS_FILES_NAME" + local out="[]" + if [ ! -f "$legacy" ] || [ -L "$legacy" ]; then + printf '%s' "$out" + return 0 + fi + local seen="" + if [ -f "$json" ] && [ ! -L "$json" ]; then + seen=$(jq -r '.external // [] | .[] | ((if .type == "properties" then "gradle-properties" else .type end) + "|" + .path)' "$json") + fi + local t p k s found jtype + while IFS=$'\t' read -r t p k; do + [ -n "$t" ] || continue + found=0 + while IFS= read -r s; do [ "$s" = "$t|$p" ] && { found=1; break; }; done <<< "$seen" + [ "$found" -eq 1 ] && continue + jtype="$t"; [ "$t" = "gradle-properties" ] && jtype="properties" + out=$(printf '%s' "$out" | jq --arg type "$jtype" --arg path "$p" --arg keys "$k" \ + '. + [if $type == "file" then {type: $type, path: $path} + else {type: $type, path: $path, keys: ($keys | split(" ") | map(select(length > 0)))} end]') + done < <(_parse_secrets_files_manifest "$legacy" 2>/dev/null) + printf '%s' "$out" +} + # Quietly emit "ws-dir/basename" for every env file in a package.json # workspace under . Emits nothing (and never dies) when is # not a workspace monorepo or jq is unavailable — plain `push` calls this @@ -1178,20 +1309,39 @@ cmd_push() { fi # ── Manifest write AFTER successful encryption (bootstrap ordering) ── - if [ "$count" -gt 0 ] && [ -n "$to_add" ] && [ "$frozen" = false ] \ - && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then - local add_json - add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') + # Two independent reasons to write: dotenv auto-adds, and absorbing a + # legacy .secrets-files (gradle-properties → properties) so the two + # external sources converge on the manifest. + local absorbed_json="[]" n_absorbed=0 + if [ "$frozen" = false ]; then + absorbed_json=$(_legacy_absorb_json "$PWD") + n_absorbed=$(printf '%s' "$absorbed_json" | jq 'length') + fi + local write_adds=false + if [ -n "$to_add" ] && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then + write_adds=true + fi + if [ "$did" -eq 1 ] && [ "$frozen" = false ] \ + && { [ "$write_adds" = true ] || [ "$n_absorbed" -gt 0 ]; }; then + local add_json="[]" + [ "$write_adds" = true ] && add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') if [ "$have_manifest" = true ]; then - jq --argjson add "$add_json" '.dotenv = ((.dotenv // []) + $add)' "$manifest" \ + jq --argjson add "$add_json" --argjson ext "$absorbed_json" \ + '.dotenv = ((.dotenv // []) + $add) | .external = ((.external // []) + $ext)' "$manifest" \ | _write_manifest_canonical "$manifest" || die "Failed to update $manifest" else - jq -n --argjson add "$add_json" '{version: '"$MANIFEST_VERSION"', dotenv: $add}' \ + jq -n --argjson add "$add_json" --argjson ext "$absorbed_json" \ + '{version: '"$MANIFEST_VERSION"', dotenv: $add} | if ($ext | length) > 0 then .external = $ext else . end' \ | _write_manifest_canonical "$manifest" || die "Failed to write $manifest" fi - while IFS= read -r e; do - [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" - done <<< "$to_add" + if [ "$write_adds" = true ]; then + while IFS= read -r e; do + [ -n "$e" ] && info "Added '$e' to $SECRETS_JSON_NAME" + done <<< "$to_add" + fi + if [ "$n_absorbed" -gt 0 ]; then + info "Absorbed $n_absorbed entr(y/ies) from $SECRETS_FILES_NAME into $SECRETS_JSON_NAME (gradle-properties → properties). $SECRETS_FILES_NAME can be deleted." + fi info "Commit the manifest so other machines pick it up. To undo an entry: edit $SECRETS_JSON_NAME (or use 'secrets push --frozen' to skip auto-add)." fi diff --git a/test/manifest.bats b/test/manifest.bats index 9800c72..d9e97ae 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -244,3 +244,138 @@ load test_helper [ "$status" -eq 1 ] [[ "$output" == *"project-relative"* ]] || false } + +# ─── C: legacy absorb + external entries via .secrets.json ───────────── + +# Local fixtures (mirror secrets.bats EGB-531/652 helpers) +m_gradle_src() { mkdir -p "$HOME/.gradle"; printf '%s' "$1" > "$HOME/.gradle/gradle.properties"; } +m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > "$HOME/keystores/upload.keystore"; } + +@test "push absorbs .secrets-files into .secrets.json (properties + file)" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + m_file_src + local dir="$WORK_DIR/absorbproj"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\nfile ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [[ "$output" == *"Absorbed"* ]] || false + run jq -r '.external | length' .secrets.json + [ "$output" = "2" ] + run jq -r '.external[] | select(.path == "~/.gradle/gradle.properties") | .type' .secrets.json + [ "$output" = "properties" ] + run jq -r '.external[] | select(.type == "file") | .path' .secrets.json + [ "$output" = "~/keystores/upload.keystore" ] + # stage 1: blob naming stays legacy-compatible + run bash -c "ls $SECRETS_DIR/absorbproj/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "absorb is idempotent — second push adds no duplicate externals" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/absorb2"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" push + [ "$status" -eq 0 ] + run jq -r '.external | length' .secrets.json + [ "$output" = "1" ] +} + +@test "external properties entry in .secrets.json drives push without .secrets-files" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/jsonextproj"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push jsonextproj + [ "$status" -eq 0 ] + [[ "$output" == *"Extracted 1 key"* ]] || false + run bash -c "ls $SECRETS_DIR/jsonextproj/external/*.gradle-properties.age" + [ "$status" -eq 0 ] +} + +@test "pull merges properties keys sourced from .secrets.json" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\nunrelated=keep\n' + local dir="$WORK_DIR/jsonpull"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + "$SECRETS_BIN" push jsonpull >/dev/null 2>&1 + m_gradle_src $'beaconClerkPkTest=STALE\nunrelated=keep\n' + run "$SECRETS_BIN" pull jsonpull + [ "$status" -eq 0 ] + run grep -c 'beaconClerkPkTest=pk_test_abc' "$HOME/.gradle/gradle.properties" + [ "$output" = "1" ] + run grep -c 'unrelated=keep' "$HOME/.gradle/gradle.properties" + [ "$output" = "1" ] +} + +@test "properties rail generalized: any *.properties basename is accepted" { + init_with_remote + mkdir -p "$HOME/.config" + printf 'apiKey=abc123\n' > "$HOME/.config/app.properties" + local dir="$WORK_DIR/genprops"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.config/app.properties","keys":["apiKey"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push genprops + [ "$status" -eq 0 ] + [[ "$output" == *"Extracted 1 key"* ]] || false +} + +@test "properties rail still blocks a non-.properties target" { + init_with_remote + printf 'PATH=/evil\n' > "$HOME/.bashrc" + local dir="$WORK_DIR/evilprops"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"properties","path":"~/.bashrc","keys":["PATH"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push evilprops + [ "$status" -eq 1 ] + [[ "$output" == *".properties"* ]] || false +} + +@test "file entry via .secrets.json round-trips binary with mode 600" { + init_with_remote + m_file_src + local dir="$WORK_DIR/jsonfile"; mkdir -p "$dir" + printf '{"version":2,"external":[{"type":"file","path":"~/keystores/upload.keystore"}]}\n' > "$dir/.secrets.json" + cd "$dir" + "$SECRETS_BIN" push jsonfile >/dev/null 2>&1 + cp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.keystore" + rm "$HOME/keystores/upload.keystore" + run "$SECRETS_BIN" pull jsonfile + [ "$status" -eq 0 ] + cmp -s "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.keystore" + local mode + mode=$(stat -f '%Lp' "$HOME/keystores/upload.keystore" 2>/dev/null || stat -c '%a' "$HOME/keystores/upload.keystore") + [ "$mode" = "600" ] +} + +@test "json file entry with keys is rejected with a warning" { + init_with_remote + m_file_src + local dir="$WORK_DIR/badfile"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"file","path":"~/keystores/upload.keystore","keys":["nope"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push badfile + [ "$status" -eq 0 ] + [[ "$output" == *"no keys"* ]] || false + run bash -c "ls $SECRETS_DIR/badfile/external/*.file.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "pull warns that .secrets-files is superseded when .secrets.json exists" { + init_with_remote + create_project_dir superproj + "$SECRETS_BIN" push superproj >/dev/null 2>&1 + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + run "$SECRETS_BIN" pull superproj + [ "$status" -eq 0 ] + [[ "$output" == *"superseded"* ]] || false +} diff --git a/test/secrets.bats b/test/secrets.bats index 56f051e..5537a03 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1217,15 +1217,17 @@ gradle_project() { [ "$mode" = "600" ] } -@test "EGB-531: target with wrong basename is refused" { +@test "EGB-531: target with non-.properties basename is refused" { + # EGB-677 generalized the rail from exact 'gradle.properties' to any + # '*.properties' basename — shell rc files and gitconfig stay blocked. init_with_remote mkdir -p "$HOME/.gradle" - printf 'beaconClerkPkTest=x\n' > "$HOME/.gradle/custom.properties" + printf 'beaconClerkPkTest=x\n' > "$HOME/.gradle/evil.sh" mkdir -p "$WORK_DIR/gproj"; cd "$WORK_DIR/gproj" - printf 'gradle-properties ~/.gradle/custom.properties beaconClerkPkTest\n' > .secrets-files + printf 'gradle-properties ~/.gradle/evil.sh beaconClerkPkTest\n' > .secrets-files run "$SECRETS_BIN" push gproj [ "$status" -eq 1 ] - [[ "$output" == *"gradle.properties"* ]] || false + [[ "$output" == *".properties"* ]] || false } @test "EGB-531: target outside HOME is refused" { @@ -1404,6 +1406,9 @@ gradle_project() { init_with_remote create_project_dir gproj "$SECRETS_BIN" push gproj >/dev/null 2>&1 + # EGB-677: drop the bootstrap .secrets.json so the legacy manifest path + # is exercised (with a manifest present, .secrets-files is superseded). + rm -f "$WORK_DIR/gproj/.secrets.json" printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$WORK_DIR/gproj/.secrets-files" cd "$WORK_DIR/gproj" run "$SECRETS_BIN" pull gproj From 0049584d9b90ae70d3ace7d96ebfe6c4367c002f Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:49:02 -0700 Subject: [PATCH 05/12] =?UTF-8?q?feat:=20manifest-driven=20pull=20?= =?UTF-8?q?=E2=80=94=20nested=20restore,=20restore-time=20rail,=20empty=20?= =?UTF-8?q?no-op=20(EGB-677=20stage=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pull with .secrets.json restores exactly the declared entries (nested paths get mkdir -p); stray store blobs are not restored - dotenv rail re-runs at restore time: unsafe entries warn+skip (pull never dies on one bad entry), missing blobs warn with a directed hint - empty manifest = warn no-op instead of a confusing 'not found' death - manifest-less projects keep the legacy glob pull verbatim --- secrets | 48 +++++++++++++++++++++++++++ test/manifest.bats | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/secrets b/secrets index e02e636..1e75d98 100755 --- a/secrets +++ b/secrets @@ -1414,6 +1414,54 @@ cmd_pull() { git -C "$SECRETS_DIR" pull >/dev/null 2>&1 fi + # ── Manifest-driven pull (EGB-677 stage 1) ── + # With a .secrets.json present, the manifest decides what restores and + # where (nested entries get their directories created). The dotenv rail + # runs again at restore time — warn+skip on pull, never die, so one bad + # entry can't block the rest of the restore. + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ -e "$manifest" ]; then + _check_manifest_file "$manifest" + local declared n_external + declared=$(jq -r '.dotenv // [] | .[]' "$manifest") + n_external=$(jq -r '.external // [] | length' "$manifest") + + if [ -z "$declared" ] && [ "$n_external" -eq 0 ]; then + echo "WARNING: $SECRETS_JSON_NAME declares nothing to pull (empty manifest). Run 'secrets push' on a machine that has the files." >&2 + ensure_store_protections + return 0 + fi + if [ -n "$declared" ] && [ ! -d "$SECRETS_DIR/$project" ]; then + die "Project '$project' not found. Run: secrets list" + fi + + local count=0 rel + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then + echo "WARNING: skipping unsafe dotenv path from $SECRETS_JSON_NAME: $rel" >&2 + continue + fi + local blob="$SECRETS_DIR/$project/${rel}.age" + if [ ! -f "$blob" ]; then + echo "WARNING: '$rel' is declared in $SECRETS_JSON_NAME but has no encrypted data in the store yet. Run 'secrets push' on a machine that has it. Skipping." >&2 + continue + fi + case "$rel" in */*) mkdir -p "$target_dir/$(dirname "$rel")" ;; esac + age -d -i "$KEY_FILE" -o "$target_dir/$rel" "$blob" + if [ ! -s "$target_dir/$rel" ]; then + echo "WARNING: Decrypted file '$rel' is empty (possibly truncated .age blob)" + fi + count=$((count + 1)) + done <<< "$declared" + + info "Decrypted $count file(s) into $target_dir" + pull_external_files "$PWD" "$project" + ensure_store_protections + return 0 + fi + + # ── Legacy glob pull (manifest-less projects; unchanged) ── # Check project exists if [ ! -d "$SECRETS_DIR/$project" ]; then die "Project '$project' not found. Run: secrets list" diff --git a/test/manifest.bats b/test/manifest.bats index d9e97ae..f23a5aa 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -379,3 +379,86 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ "$status" -eq 0 ] [[ "$output" == *"superseded"* ]] || false } + +# ─── D: pull from manifest — nested restore, restore-time rail ───────── + +@test "pull restores manifest-declared nested file (mkdir -p)" { + init_with_remote + create_project_dir nestpull + mkdir -p packages/web + echo "K=v" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + rm -rf packages + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [ -f packages/web/.env.development ] + [ "$(cat packages/web/.env.development)" = "K=v" ] +} + +@test "pull with manifest restores only declared entries" { + init_with_remote + create_project_dir onlydecl + "$SECRETS_BIN" push >/dev/null 2>&1 + # plant an undeclared stray blob in the store + local pubkey; pubkey=$(age-keygen -y "$SECRETS_DIR/key.txt") + echo "S=1" | age -r "$pubkey" -o "$SECRETS_DIR/onlydecl/.env.stray.age" + rm -f .env .env.staging + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [ -f .env ] + [ ! -f .env.stray ] +} + +@test "pull warns and skips an unsafe manifest entry, restores the rest" { + init_with_remote + create_project_dir unsafepull + "$SECRETS_BIN" add .env >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + printf '{"version":2,"dotenv":[".env","../escape/.env"]}\n' > .secrets.json + rm -f .env + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [[ "$output" == *"skipping unsafe"* ]] || false + [ -f .env ] + [ ! -f "$WORK_DIR/escape/.env" ] +} + +@test "pull on an empty manifest is a warn no-op" { + init_with_remote + mkdir -p "$WORK_DIR/emptypull" + cd "$WORK_DIR/emptypull" + printf '{"version":2,"dotenv":[]}\n' > .secrets.json + run "$SECRETS_BIN" pull emptypull + [ "$status" -eq 0 ] + [[ "$output" == *"declares nothing"* ]] || false +} + +@test "pull warns when a declared entry has no blob in the store" { + init_with_remote + create_project_dir nopullblob + "$SECRETS_BIN" push >/dev/null 2>&1 + jq '.dotenv += [".env.missing"]' .secrets.json > .secrets.json.tmp && mv .secrets.json.tmp .secrets.json + run "$SECRETS_BIN" pull + [ "$status" -eq 0 ] + [[ "$output" == *".env.missing"* ]] || false + [[ "$output" == *"no encrypted data"* ]] || false +} + +@test "machine-2 flow: committed manifest + pull restores everything" { + init_with_remote + create_project_dir machine1 + mkdir -p packages/api + echo "API=1" > packages/api/.dev.vars + "$SECRETS_BIN" add packages/api/.dev.vars >/dev/null + "$SECRETS_BIN" push m2proj >/dev/null 2>&1 + # simulate machine 2: fresh dir, only the committed manifest present + mkdir -p "$WORK_DIR/machine2" + cp .secrets.json "$WORK_DIR/machine2/" + cd "$WORK_DIR/machine2" + run "$SECRETS_BIN" pull m2proj + [ "$status" -eq 0 ] + [ -f .env ] + [ -f packages/api/.dev.vars ] + [ "$(cat packages/api/.dev.vars)" = "API=1" ] +} From 89e851278b2db645591975827d46c9db7c538408 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 08:58:16 -0700 Subject: [PATCH 06/12] feat: jq gating, platform-aware install hints, stage-1 docs (EGB-677 stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jq required only when .secrets.json exists or is being written; manifest-less projects run jq-free (features skipped with a notice) — clone-and-run for v1 users survives (no-jq fixture excludes /usr/bin, macOS ships jq there now) - check_cmd: platform-aware hints (brew/apt-get/dnf/generic) instead of hardcoded brew — correct guidance on Linux/CI - cmd_help: add command, push flags, manifest section with example - README: manifest section, external files rewritten around .secrets.json (legacy .secrets-files documented as absorbed), troubleshooting entries, command table, test instructions - CLAUDE.md: manifest architecture notes, bash-3.2 '[[ ]] || false' testing convention, project structure refresh --- CLAUDE.md | 13 ++++++-- README.md | 77 +++++++++++++++++++++++++++++++++++++--------- secrets | 55 ++++++++++++++++++++++++++++++--- test/manifest.bats | 45 +++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0da5504..ede0401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,9 +16,14 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core -bats test/secrets.bats +bats test/ # runs secrets.bats + manifest.bats ``` +**bash 3.2 assertion gotcha:** bats runs under system bash 3.2, where a +failing `[[ ]]` mid-test does NOT fail the test (the ERR trap skips `[[` +compound commands). Every standalone `[[ ... ]]` assertion MUST end with +`|| false`. Single-bracket `[ ]` assertions are unaffected. + ## Architecture Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. @@ -26,6 +31,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) +- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` - Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`) @@ -34,10 +40,11 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek ## Project Structure ``` -secrets # CLI script (~600 lines bash) +secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ - secrets.bats # bats-core test suite (126 tests) + secrets.bats # bats-core test suite (133 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (41 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index d68e566..f7e754d 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,10 @@ secrets clear |---------|-------------| | `secrets init` | Create the `~/.secrets/` repo and generate an encryption key | | `secrets push` | Encrypt secret files in the current directory and upload them | +| `secrets push --frozen` | Sync only what `.secrets.json` declares (skip auto-add) | +| `secrets push --dry-run` | Show what would be added/synced without changing anything | | `secrets pull` | Download and decrypt secret files into the current directory | +| `secrets add ` | Declare a project-relative file in `.secrets.json` | | `secrets clear` | Delete plaintext secret files from the current directory | | `secrets run ` | Pull secrets, run a command, then clear secrets when it exits | | `secrets list` | Show all projects that have stored secrets | @@ -176,6 +179,32 @@ When you run `secrets push` or `secrets pull` without specifying a project name, You can also specify a name explicitly: `secrets push myapp`. +### The manifest (`.secrets.json`) + +Every project gets a committed `.secrets.json` at its root declaring exactly what syncs — the single source of truth `push` and `pull` operate from (requires `jq`): + +```json +{ + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", ".env.staging", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] +} +``` + +You rarely write it by hand: + +- **Auto-add (default):** `secrets push` discovers conventional files (`.env`, `.env.*`, `.dev.vars` — plus `package.json` workspace dirs once a manifest exists) and adds them to the manifest with an `==>` notice. Commit the manifest so other machines pick it up. +- **Explicit mode:** set `"options": {"autoAdd": false}` (a committed, team-shared setting) and `push` only syncs declared entries, warning about undeclared files. `secrets add ` is then the only manifest writer. Per-invocation: `push --frozen` (declared-only once) and `push --dry-run` (preview). +- `dotenv` paths are project-relative — nested monorepo paths like `packages/@acme/web/.env` are welcome; `..`, absolute paths, and symlinked manifests are refused. +- On the other machine, `secrets pull` restores exactly what the committed manifest declares, creating nested directories as needed. + +Projects without a manifest keep working exactly as before (and work without `jq`); the first `push` bootstraps one for you. + ### secrets run `secrets run` is a **pull → run → clear** pipeline: it runs `secrets pull` to decrypt the latest files into your project, executes your command, then runs `secrets clear` when that command finishes. Plaintext `.env` / `.dev.vars` files exist only while your command is running. @@ -352,25 +381,40 @@ Requires `jq` (`brew install jq`). Some credentials don't live in your project at all. Android builds, for example, read keys from `~/.gradle/gradle.properties` — a global file, outside any project, shared by every Gradle project on your machine (the project's own `gradle.properties` is git-tracked, so it's the wrong home for secrets). `secrets` can sync specific keys from such a file without touching the unrelated keys around them. -You declare what to sync in a committed `.secrets-files` manifest at your project root, one entry per line: +You declare what to sync in the `external` array of your committed `.secrets.json`: -``` -# -gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive +```json +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} ``` -- **type** — `gradle-properties` (sync named keys) or `file` (sync the whole file — see below). -- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `gradle-properties` the basename must be `gradle.properties`. -- **keys** — the property names to sync (`gradle-properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. +- **type** — `properties` (sync named keys from a Java-properties-style file) or `file` (sync the whole file — see below). +- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `properties` the basename must end in `.properties`. +- **keys** — the property names to sync (`properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. + +> **Legacy `.secrets-files`:** older projects declared these entries in a line-based `.secrets-files`. It still parses, and the next `secrets push` absorbs its entries into `.secrets.json` (type `gradle-properties` becomes `properties`) — after that the legacy file is superseded and can be deleted. #### Syncing to a second machine -On the machine that already has the keys set: +On the machine that already has the keys set, add the entry to `.secrets.json` (create the file if the project doesn't have one yet): ```bash cd ~/myapp -echo "gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive" > .secrets-files -git add .secrets-files && git commit -m "sync gradle Clerk keys" +cat > .secrets.json <<'EOF' +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} +EOF +git add .secrets.json && git commit -m "sync gradle Clerk keys" secrets push # ==> Extracted 2 key(s) from ~/.gradle/gradle.properties ``` @@ -393,9 +437,8 @@ secrets pull Some external secrets are whole binary files — an Android upload keystore, a certificate. The `file` type syncs the file verbatim (binary-safe, encrypted with age like everything else): -``` -# -file ~/keystores/beacon-upload.keystore +```json +{ "type": "file", "path": "~/keystores/beacon-upload.keystore" } ``` On `secrets push` the file is encrypted into `/external/`. On `secrets pull` it is restored to the same path with mode `600`; if a different version already exists there, it is backed up to `.secrets-bak` first. The same path rules apply (inside `$HOME`, no `..`, no symlinks). Like merged Gradle keys, restored files are permanent plaintext on disk — `secrets clear` does not remove them. @@ -444,10 +487,14 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` **"Fast-forward pull failed"** — Someone else pushed secrets while you had local changes. Run `secrets pull` first, then retry your push. +**".secrets.json: invalid JSON" / "manifest version N is not supported"** — The committed manifest is malformed or written by a newer `secrets`. The error names the file; fix the syntax, or update the tool (`git pull` in the tool's clone). + +**"'jq' is not installed"** — Manifest features need `jq`. The error prints the install command for your platform. Manifest-less projects work without it. + ## Development ```bash -# Run the test suite (126 tests) +# Run the test suite (174 tests across both files) brew install bats-core -bats test/secrets.bats +bats test/ ``` diff --git a/secrets b/secrets index 1e75d98..7e01e91 100755 --- a/secrets +++ b/secrets @@ -37,7 +37,18 @@ die() { echo "ERROR: $*" >&2; exit 1; } info() { echo "==> $*"; } check_cmd() { - command -v "$1" >/dev/null 2>&1 || die "'$1' is not installed. Run: brew install $1" + command -v "$1" >/dev/null 2>&1 && return + # Platform-aware install hint — hardcoding brew is wrong guidance on a + # Linux box or CI runner (EGB-677 DX review). + local hint="install '$1' with your package manager" + if command -v brew >/dev/null 2>&1; then + hint="brew install $1" + elif command -v apt-get >/dev/null 2>&1; then + hint="sudo apt-get install $1" + elif command -v dnf >/dev/null 2>&1; then + hint="sudo dnf install $1" + fi + die "'$1' is not installed. Run: $hint" } check_initialized() { @@ -1216,6 +1227,12 @@ cmd_push() { pubkey=$(get_pubkey) # ── Manifest read (validated; absence = bootstrap) ── + # jq is required only when a manifest exists (authoritative, can't be + # ignored) or is being written. Without jq on a manifest-less project, + # manifest features are skipped with a notice — clone-and-run for v1 + # users survives. + local have_jq=true + command -v jq >/dev/null 2>&1 || have_jq=false local manifest="$PWD/$SECRETS_JSON_NAME" local have_manifest=false auto_add=true declared="" if [ -e "$manifest" ]; then @@ -1312,8 +1329,11 @@ cmd_push() { # Two independent reasons to write: dotenv auto-adds, and absorbing a # legacy .secrets-files (gradle-properties → properties) so the two # external sources converge on the manifest. + if [ "$have_jq" = false ]; then + echo "NOTE: jq not found — skipping $SECRETS_JSON_NAME manifest features (auto-add, absorb). Install jq to enable them." >&2 + fi local absorbed_json="[]" n_absorbed=0 - if [ "$frozen" = false ]; then + if [ "$frozen" = false ] && [ "$have_jq" = true ]; then absorbed_json=$(_legacy_absorb_json "$PWD") n_absorbed=$(printf '%s' "$absorbed_json" | jq 'length') fi @@ -1321,7 +1341,7 @@ cmd_push() { if [ -n "$to_add" ] && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then write_adds=true fi - if [ "$did" -eq 1 ] && [ "$frozen" = false ] \ + if [ "$did" -eq 1 ] && [ "$frozen" = false ] && [ "$have_jq" = true ] \ && { [ "$write_adds" = true ] || [ "$n_absorbed" -gt 0 ]; }; then local add_json="[]" [ "$write_adds" = true ] && add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') @@ -1905,21 +1925,48 @@ secrets — encrypted secret file sync between machines Usage: secrets init Initialize the secrets repo and generate an age key secrets push [project] Encrypt secret files and push to the secrets repo + secrets push --frozen Sync only manifest-declared files (skip auto-add) + secrets push --dry-run Show what would be added/synced; change nothing secrets push -w|--workspaces Push secrets from all workspaces in package.json secrets pull [project] Pull and decrypt secret files into current directory secrets pull -w|--workspaces Pull secrets into all workspaces from package.json + secrets add Declare a project-relative file in .secrets.json secrets clear Remove plaintext secret files from current directory secrets clear -w|--workspaces Clear secrets from all workspaces in package.json secrets run [-w] Pull secrets, run command, clear secrets on exit secrets list List all projects and their secret files secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key - secrets which Show the active store path and which rule chose it + secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` Tracked files: .env, .env.*, .dev.vars +Manifest (.secrets.json): + A committed project-root manifest declaring everything the project + syncs (requires jq). `secrets push` discovers conventional files and + auto-adds them with a notice; set {"options":{"autoAdd":false}} to + require explicit `secrets add` instead. Dotenv paths are project- + relative (nested workspace paths welcome); external entries use + {"type":"properties"|"file","path":...,"keys":[...]}: + + { + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] + } + + A legacy .secrets-files is absorbed into .secrets.json on the next + push (gradle-properties entries become type "properties") and can be + deleted afterwards. Without jq, manifest-less projects keep working; + manifest features are skipped with a notice. + If [project] is omitted, it is derived from the current directory's git remote (if available) or the directory name. diff --git a/test/manifest.bats b/test/manifest.bats index f23a5aa..822c1ca 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -462,3 +462,48 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ -f packages/api/.dev.vars ] [ "$(cat packages/api/.dev.vars)" = "API=1" ] } + +# ─── E: jq gating + install hints + help ─────────────────────────────── + +# Helper: PATH with age but without jq. macOS ships /usr/bin/jq, so +# /usr/bin must be excluded too — needed tools are symlinked explicitly. +m_nojq_path() { + local fake="$TEST_TMPDIR/nojq-bin" + mkdir -p "$fake" + local t + for t in age age-keygen git basename dirname mktemp grep sed tr cut cksum stat head tail sort uniq wc env touch find diff cmp; do + command -v "$t" >/dev/null 2>&1 && ln -sf "$(command -v "$t")" "$fake/$t" + done + rm -f "$fake/jq" + echo "$fake:/bin" +} + +@test "manifest-less push works without jq (manifest features skipped)" { + init_with_remote + create_project_dir nojqproj + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/nojqproj/.env.age" ] + [ ! -f ".secrets.json" ] + [[ "$output" == *"jq"* ]] || false +} + +@test "push dies with an install hint when a manifest exists but jq is missing" { + init_with_remote + create_project_dir needjq + printf '{"version":2,"dotenv":[".env"]}\n' > .secrets.json + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [[ "$output" == *"'jq' is not installed"* ]] || false +} + +@test "help documents add, --frozen, --dry-run and the manifest" { + run "$SECRETS_BIN" help + [ "$status" -eq 0 ] + [[ "$output" == *"secrets add"* ]] || false + [[ "$output" == *"--frozen"* ]] || false + [[ "$output" == *"--dry-run"* ]] || false + [[ "$output" == *".secrets.json"* ]] || false +} From ecc4f2349484e719a6ae8b18d1ef7f7bc9bffc3b Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 09:40:43 -0700 Subject: [PATCH 07/12] test: coverage for stage-1 gap paths (json rail warns, symlink legacy, directed errors, dry-run/which branches) --- test/manifest.bats | 115 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/test/manifest.bats b/test/manifest.bats index 822c1ca..c4d7314 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -507,3 +507,118 @@ m_nojq_path() { [[ "$output" == *"--dry-run"* ]] || false [[ "$output" == *".secrets.json"* ]] || false } + +# ─── Coverage backfill (ship Step 7 gap paths) ───────────────────────── + +@test "json external entry with unknown type warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unktype"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"wat","path":"~/x.properties","keys":["k"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unktype + [ "$status" -eq 0 ] + [[ "$output" == *"unknown external type"* ]] || false +} + +@test "json external entry with unsafe path warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unsafext"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/../etc/x.properties","keys":["k"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unsafext + [ "$status" -eq 0 ] + [[ "$output" == *"unsafe characters in external path"* ]] || false +} + +@test "json external entry with unsafe keys warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/unsafekeys"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["k;rm"]}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push unsafekeys + [ "$status" -eq 0 ] + [[ "$output" == *"unsafe characters in key list"* ]] || false +} + +@test "json properties entry without keys warns and is skipped" { + init_with_remote + local dir="$WORK_DIR/nokeys"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"external":[{"type":"properties","path":"~/.gradle/gradle.properties"}]}\n' > "$dir/.secrets.json" + cd "$dir" + run "$SECRETS_BIN" push nokeys + [ "$status" -eq 0 ] + [[ "$output" == *"has no keys"* ]] || false +} + +@test "symlinked .secrets-files is ignored with a warning on push" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=x\n' + local dir="$WORK_DIR/symlegacy"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$TEST_TMPDIR/real-sf" + ln -s "$TEST_TMPDIR/real-sf" "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push symlegacy + [ "$status" -eq 0 ] + [[ "$output" == *"symlink"* ]] || false + run bash -c "ls $SECRETS_DIR/symlegacy/external/*.age 2>/dev/null" + [ "$status" -ne 0 ] +} + +@test "pull with declared entries dies with directed error when project absent from store" { + init_with_remote + mkdir -p "$WORK_DIR/ghostproj" + cd "$WORK_DIR/ghostproj" + printf '{"version":2,"dotenv":[".env"]}\n' > .secrets.json + run "$SECRETS_BIN" pull ghostproj + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] || false + [[ "$output" == *"secrets list"* ]] || false +} + +@test "add without an argument dies with usage" { + create_project_dir noargadd + run "$SECRETS_BIN" add + [ "$status" -eq 1 ] + [[ "$output" == *"Usage: secrets add"* ]] || false +} + +@test "push rejects an unknown flag with usage" { + init_with_remote + create_project_dir badflag + run "$SECRETS_BIN" push --nope + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown push flag"* ]] || false +} + +@test "add normalizes a leading ./ prefix" { + create_project_dir dotslash + run "$SECRETS_BIN" add ./.env + [ "$status" -eq 0 ] + run jq -r '.dotenv[0]' .secrets.json + [ "$output" = ".env" ] +} + +@test "push --dry-run reports nothing-new when manifest covers all discovered files" { + init_with_remote + create_project_dir alldecl + "$SECRETS_BIN" add .env >/dev/null + "$SECRETS_BIN" add .env.staging >/dev/null + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Nothing new to add"* ]] || false +} + +@test "which displays external entries from the manifest" { + create_project_dir whichext + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["k1"]}]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"properties"* ]] || false + [[ "$output" == *"gradle.properties"* ]] || false + [[ "$output" == *"k1"* ]] || false +} From 5489f894460fb1329d54073cbd134b71e7332e5d Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 10:17:07 -0700 Subject: [PATCH 08/12] =?UTF-8?q?test:=20coverage=20backfill=20=E2=80=94?= =?UTF-8?q?=20which=20UNSAFE=20marker,=20malformed-manifest=20add,=20dry-r?= =?UTF-8?q?un=20declared=20list,=20frozen=20absorb=20suppression,=20file-t?= =?UTF-8?q?ype=20absorb=20round-trip,=20external=20dedup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/manifest.bats | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/test/manifest.bats b/test/manifest.bats index c4d7314..e273479 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -622,3 +622,79 @@ m_nojq_path() { [[ "$output" == *"gradle.properties"* ]] || false [[ "$output" == *"k1"* ]] || false } + +# ─── F: ship Step 7 coverage backfill (audit gaps) ───────────────────── + +@test "which flags an unsafe dotenv entry with the UNSAFE marker" { + create_project_dir whichunsafe + printf '{"version":2,"dotenv":[".env","../escape/.env"]}\n' > .secrets.json + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"UNSAFE"* ]] || false + [[ "$output" == *"will be refused"* ]] || false +} + +@test "add to a malformed existing manifest dies with a directed error" { + create_project_dir addmalformed + echo '{ not json' > .secrets.json + run "$SECRETS_BIN" add .env + [ "$status" -eq 1 ] + [[ "$output" == *"invalid"* ]] || false +} + +@test "push --dry-run lists declared entries under 'Would sync'" { + init_with_remote + create_project_dir drysync + "$SECRETS_BIN" add .env >/dev/null + run "$SECRETS_BIN" push --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Would sync (declared)"* ]] || false + [[ "$output" == *".env"* ]] || false +} + +@test "push --frozen does not absorb a legacy .secrets-files" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/frozenabsorb"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"dotenv":[".env"]}\n' > "$dir/.secrets.json" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push --frozen frozenabsorb + [ "$status" -eq 0 ] + [[ "$output" != *"Absorbed"* ]] || false + run jq -r '.external // [] | length' .secrets.json + [ "$output" = "0" ] +} + +@test "file entry absorbed from legacy round-trips on pull" { + init_with_remote + m_file_src + local dir="$WORK_DIR/fileabsorb"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf 'file ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + "$SECRETS_BIN" push fileabsorb >/dev/null 2>&1 + run jq -r '.external[] | select(.type=="file") | .path' .secrets.json + [ "$output" = "~/keystores/upload.keystore" ] + cp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.ks" + rm "$HOME/keystores/upload.keystore" + rm -f .secrets-files + run "$SECRETS_BIN" pull fileabsorb + [ "$status" -eq 0 ] + cmp -s "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/orig.ks" +} + +@test "push dedups a legacy entry already present in the manifest external[]" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + local dir="$WORK_DIR/dedupext"; mkdir -p "$dir" + echo "K=v" > "$dir/.env" + printf '{"version":2,"dotenv":[".env"],"external":[{"type":"properties","path":"~/.gradle/gradle.properties","keys":["beaconClerkPkTest"]}]}\n' > "$dir/.secrets.json" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push dedupext + [ "$status" -eq 0 ] + run jq -r '.external | length' .secrets.json + [ "$output" = "1" ] +} From c6ea724ddbd36b002b75a15bc1a1497c04fcd9f6 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:12:27 -0700 Subject: [PATCH 09/12] fix: rekey and list recurse into nested manifest blobs (EGB-677 stage 1) Pre-landing review (testing + checklist specialists, reproduced) caught a data-loss bug: cmd_rekey's decrypt/re-encrypt globs were non-recursive and only special-cased external/. Nested manifest dotenv blobs (/.age, new this branch) were never visited, so after a key rotation they stayed encrypted under the discarded old key = permanently undecryptable. cmd_list had the same blind spot (cosmetic: nested entries invisible in listings). Both now walk the entire project tree with `find -type f` (bash 3.2 safe, includes dotfiles natively), unifying top-level / nested / external blobs into one recursive pass and dropping the now-redundant external/ special-casing. Regression tests: nested-blob rekey round-trip (survives rotation) + list shows nested entry. Full suite 193/193. --- secrets | 85 ++++++++++++++++++++-------------------------- test/manifest.bats | 33 ++++++++++++++++++ 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/secrets b/secrets index 7e01e91..ba2adce 100755 --- a/secrets +++ b/secrets @@ -1607,19 +1607,20 @@ cmd_list() { # Skip hidden dirs [[ "$project" == .* ]] && continue echo "$project:" - for f in "$dir"*.age "$dir".*.age; do + # Recurse the whole project tree so nested manifest blobs + # (/.age) are visible, not just top-level entries. + # External blobs (external/.age) are labelled distinctly. + while IFS= read -r f; do [ -f "$f" ] || continue - echo " $(basename "$f" .age)" + local rel + rel=${f#"$dir"} + rel=${rel%.age} + case "$rel" in + external/*) echo " [external] ${rel#external/}" ;; + *) echo " $rel" ;; + esac found=1 - done - # External files live in a subdir, invisible to the globs above. - if [ -d "${dir}external" ]; then - for f in "${dir}external"/*.age; do - [ -f "$f" ] || continue - echo " [external] $(basename "$f" .age)" - found=1 - done - fi + done < <(find "$dir" -type f -name '*.age' | sort) done if [ "$found" -eq 0 ]; then @@ -1681,29 +1682,23 @@ cmd_rekey() { project=$(basename "$dir") [[ "$project" == .* ]] && continue mkdir -p "$tmpdir/$project" - for f in "$dir"*.age "$dir".*.age; do + # Walk the WHOLE project tree, not just its top level. Manifest dotenv + # entries can nest (/.age) and external blobs live in + # /external/.age. A non-recursive glob would skip both, + # leaving them encrypted under the old key = permanently undecryptable + # after rotation (silent data loss). `find` is bash-3.2 safe and recurses. + while IFS= read -r f; do [ -f "$f" ] || continue - local name - name=$(basename "$f" .age) - if ! age -d -i "$KEY_FILE" -o "$tmpdir/$project/$name" "$f"; then - die "Decryption failed for $project/$name. Rekey aborted. Old key preserved." + local rel dest + rel=${f#"$dir"} # path relative to the project dir (keeps .age) + rel=${rel%.age} # strip the .age suffix → original relpath + dest="$tmpdir/$project/$rel" + mkdir -p "$(dirname "$dest")" + if ! age -d -i "$KEY_FILE" -o "$dest" "$f"; then + die "Decryption failed for $project/$rel. Rekey aborted. Old key preserved." fi file_count=$((file_count + 1)) - done - # External files live in a subdir; rekey them too or they become - # undecryptable after rotation. - if [ -d "${dir}external" ]; then - mkdir -p "$tmpdir/$project/external" - for f in "${dir}external"/*.age; do - [ -f "$f" ] || continue - local ename - ename=$(basename "$f" .age) - if ! age -d -i "$KEY_FILE" -o "$tmpdir/$project/external/$ename" "$f"; then - die "Decryption failed for $project/external/$ename. Rekey aborted. Old key preserved." - fi - file_count=$((file_count + 1)) - done - fi + done < <(find "$dir" -type f -name '*.age') done if [ "$file_count" -eq 0 ]; then @@ -1723,29 +1718,23 @@ cmd_rekey() { info "Re-encrypting all files with new key..." - # Re-encrypt all files. The ".*" glob is required: dotenv files decrypt - # to dotfiles ("$tmpdir/p/.env") that a bare "*" would silently skip, - # leaving their blobs on the old key (undecryptable after rotation). + # Re-encrypt all files. `find -type f` recurses into nested dotenv dirs and + # external/ and natively includes dotfiles (decrypted dotenv files like + # "$tmpdir/p/.env"), which a bare "*" glob would silently skip — leaving + # their blobs on the old key (undecryptable after rotation). The walk mirrors + # the recursive decrypt above so every blob round-trips back to its relpath. for dir in "$tmpdir"/*/; do [ -d "$dir" ] || continue local project project=$(basename "$dir") mkdir -p "$SECRETS_DIR/$project" - for f in "$dir"* "$dir".*; do + while IFS= read -r f; do [ -f "$f" ] || continue - local name - name=$(basename "$f") - age -r "$pubkey" -o "$SECRETS_DIR/$project/${name}.age" "$f" - done - if [ -d "${dir}external" ]; then - mkdir -p "$SECRETS_DIR/$project/external" - for f in "${dir}external"/*; do - [ -f "$f" ] || continue - local ename - ename=$(basename "$f") - age -r "$pubkey" -o "$SECRETS_DIR/$project/external/${ename}.age" "$f" - done - fi + local rel + rel=${f#"$dir"} # path relative to the project temp dir + mkdir -p "$(dirname "$SECRETS_DIR/$project/$rel")" + age -r "$pubkey" -o "$SECRETS_DIR/$project/${rel}.age" "$f" + done < <(find "$dir" -type f) done # Commit and push (heal .gitignore first so add -A can't stage key.txt) diff --git a/test/manifest.bats b/test/manifest.bats index e273479..4459a6c 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -698,3 +698,36 @@ m_nojq_path() { run jq -r '.external | length' .secrets.json [ "$output" = "1" ] } + +# ─── J: rekey + list recurse into nested manifest blobs (data-loss guard) ────── + +@test "rekey re-encrypts a nested manifest dotenv blob (survives rotation)" { + # Regression: cmd_rekey's non-recursive glob skipped /.age + # blobs, leaving them on the old key = permanently undecryptable after rotation. + init_with_remote + create_project_dir nestrekey + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + [ -f "$SECRETS_DIR/nestrekey/packages/web/.env.development.age" ] + run "$SECRETS_BIN" rekey + [ "$status" -eq 0 ] + rm -rf packages + run "$SECRETS_BIN" pull nestrekey + [ "$status" -eq 0 ] + [ -f packages/web/.env.development ] + [ "$(cat packages/web/.env.development)" = "N=nested" ] +} + +@test "list shows a nested manifest blob" { + init_with_remote + create_project_dir nestlist + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" list + [ "$status" -eq 0 ] + [[ "$output" == *"packages/web/.env.development"* ]] || false +} From 588f290dcc131c91b2ce1d6fc02463e43c36634c Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:12:39 -0700 Subject: [PATCH 10/12] chore: security review policy + operator runner, doc updates (EGB-677 stage 1) Add .ship-policy.json (opts out AI adversarial/red-team/security-specialist review; requires local operator sign-off) and test/run-security.sh (the operator-local security regression subset). Document the policy in CLAUDE.md and README, fix stale test counts (manifest.bats 41->58, total 174->191), and update the storage-recursion note to reflect rekey/list now walking the full project tree. --- .ship-policy.json | 15 ++++++++++ CLAUDE.md | 36 ++++++++++++++++++++++-- README.md | 9 +++++- test/run-security.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 .ship-policy.json create mode 100755 test/run-security.sh diff --git a/.ship-policy.json b/.ship-policy.json new file mode 100644 index 0000000..034532e --- /dev/null +++ b/.ship-policy.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "ai_review": { + "adversarial": false, + "red_team": false, + "security_specialist": false, + "reason": "This repo ships defensive security regression tests with attack-payload fixtures (shell injection, path traversal, symlink rails). AI-driven adversarial, red-team, and security-specialist review triggers Anthropic Usage Policy blocks (real-time cyber safeguards on hosted models). Do not dispatch those subagents or prompts for this repository." + }, + "manual_security_signoff": { + "script": "./test/run-security.sh", + "signoff_file": ".gstack/security-signoff.json", + "required_before_ship": true, + "instructions": "A human operator runs the script locally, reviews output, and types SIGNOFF. Agents must not run the security suite or substitute AI review for sign-off." + } +} diff --git a/CLAUDE.md b/CLAUDE.md index ede0401..174bf07 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,6 +17,7 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core bats test/ # runs secrets.bats + manifest.bats +./test/run-security.sh # security regression subset + operator sign-off (see below) ``` **bash 3.2 assertion gotcha:** bats runs under system bash 3.2, where a @@ -24,6 +25,35 @@ failing `[[ ]]` mid-test does NOT fail the test (the ERR trap skips `[[` compound commands). Every standalone `[[ ... ]]` assertion MUST end with `|| false`. Single-bracket `[ ]` assertions are unaffected. +## Security review policy (mandatory for agents) + +This repo contains **defensive** security regression tests with deliberate +attack-payload fixtures (shell injection strings, symlink escape cases, path +rails). Hosted models (including Opus) block AI-driven "think like an attacker" +/ red-team / security-specialist review of that material under Anthropic Usage +Policy (real-time cyber safeguards). + +**Read `.ship-policy.json` before any `/ship`, `/review`, or `/autoplan` work.** + +When that file is present (it always is in this repo): + +1. **Never** dispatch red-team subagents, security specialist subagents, or + adversarial review prompts ("think like an attacker", "find security holes", + "chaos engineer", exploit hunting) against this codebase. +2. **Never** run `./test/run-security.sh` or filter bats to SECURITY/adversarial + tests on behalf of the user — that is operator-local work only. +3. **Before ship/PR:** ask the human operator to run `./test/run-security.sh` + locally and complete the SIGNOFF prompt. Record lives at + `.gstack/security-signoff.json` (gitignored). Proceed with ship only after + they confirm sign-off for the current `HEAD` (or note in the PR that sign-off + is pending). +4. Normal `./ship` steps may continue (tests via `bats test/`, eng review, + changelog, PR) — only the AI security/red-team/adversarial passes are opted out. + +Gstack `/ship` reads the same policy via `gstack-ship-policy` (sources +`.ship-policy.json` at repo root): Step 9 blocks when manual sign-off is missing, +skips security specialist + red team, and Step 11 skips adversarial review. + ## Architecture Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. @@ -44,7 +74,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (41 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (58 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file @@ -74,7 +104,7 @@ The active store directory is picked by `resolve_store()` using these rules, hig Key design decisions (all driven by /autoplan review): - **Wire-in is at command scope** (`cmd_push`/`cmd_pull`), via `push_external_files` / `pull_external_files`, **not** inside `push_dir_to_project` / `pull_project_to_dir` (those loop per-workspace and `pull_project_to_dir` uses stdout as a data channel). -- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the existing non-recursive `*.age` / `.*.age` globs (pull, list, rekey), so the old dotenv path can never decrypt a blob into cwd. `cmd_rekey` and `cmd_list` recurse into `external/` explicitly (rekey MUST, or the blob is orphaned after rotation = data loss). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). +- **Storage:** blobs live in `$SECRETS_DIR//external/.gradle-properties.age`. The `external/` subdir keeps them out of the legacy non-recursive `*.age` / `.*.age` globs the dotenv `pull` path uses, so a dotenv pull can never decrypt an external blob into cwd. `cmd_rekey` and `cmd_list` instead walk the **entire** project tree (`find -type f`), so they cover both `external/.age` and nested manifest dotenv blobs (`/.age`) — rekey MUST recurse, or any nested/external blob is orphaned under the old key after rotation = data loss (EGB-677 regression test: "rekey re-encrypts a nested manifest dotenv blob"). `` = manifest path token with non-`[A-Za-z0-9._-]` chars → `_`, plus a `cksum` suffix of the original path so paths that clean to the same string (`a/b` vs `a_b`) don't collide. Machine-independent (derived from the committed manifest token, not the expanded path). - **Merge is pure bash, no `sed`/regex** (`merge_gradle_keys`): exact-string key comparison (avoids `beaconClerkPk` vs `beaconClerkPkTest` substring bug), value treated as opaque literal (survives `& \ /` in values). Updates a managed key in place at its first occurrence, collapses duplicates, appends new keys, preserves unrelated lines/comments/order. Continuation lines (trailing odd backslashes, tracked by `_trailing_bs_odd`) are never matched as keys. Atomic write: temp in the same dir → `chmod` to match (or `600` on create) → `mv`. Backs up to `.secrets-bak` before each merge. - **Properties separator parsing** (`_props_get`): key ends at the first `=`, `:`, or whitespace (after lstrip); handles `key=value`, `key = value`, `key:value`, `key value`; last definition wins. - **Security:** the write target comes from a committed file, so `_validate_external_target_path` locks it down — basename must be `gradle.properties`, must resolve inside `$HOME` (deepest-existing-ancestor resolved, symlink target/parent refused, `..` rejected). This blocks a malicious manifest from appending decrypted keys to `~/.gitconfig`/`~/.bashrc`. `_parse_secrets_files_manifest` rejects shell metacharacters/control chars in path and keys (path allows `[A-Za-z0-9/._~-]` only; keys allow `[A-Za-z0-9._-]` + space), mirrors the `.secrets-store` posture (no shell expansion, symlinked manifest skipped). @@ -119,7 +149,7 @@ Key routing rules: - QA/testing site behavior → invoke /qa or /qa-only - Code review/diff check → invoke /review - Visual polish → invoke /design-review -- Ship/deploy/PR → invoke /ship or /land-and-deploy +- Ship/deploy/PR → invoke /ship or /land-and-deploy (after reading `.ship-policy.json`; no AI adversarial/red-team/security-specialist review in this repo) - Save progress → invoke /context-save - Resume context → invoke /context-restore - Author a backlog-ready spec/issue → invoke /spec diff --git a/README.md b/README.md index f7e754d..421aad5 100644 --- a/README.md +++ b/README.md @@ -494,7 +494,14 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (174 tests across both files) +# Run the test suite (191 tests across both files) brew install bats-core bats test/ + +# Security regression subset — operator-local only (attack-payload fixtures). +# Required before ship; records sign-off in .gstack/security-signoff.json. +./test/run-security.sh ``` + +Hosted AI agents must not run the security script or perform red-team/adversarial +review on this repo — see `.ship-policy.json` and `CLAUDE.md`. diff --git a/test/run-security.sh b/test/run-security.sh new file mode 100755 index 0000000..e7168ab --- /dev/null +++ b/test/run-security.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Local-only security regression suite. Uses attack-payload fixtures on purpose. +# Do NOT ask hosted AI agents to run this script or to perform equivalent red-team review. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +if ! command -v bats >/dev/null 2>&1; then + echo "bats-core is required: brew install bats-core" >&2 + exit 1 +fi + +# Subset of the full suite: adversarial regressions + explicit SECURITY tests + +# closely related path/injection rails. Keeps the run focused and fast. +FILTER='SECURITY|F1:|F2:|F3:|F4:|F5:|command injection does not execute|outside HOME is refused|symlinked target is refused|shell metacharacters|command-substitution|symlinked .secrets-store is skipped|symlinked .secrets-files is ignored|symlinked .secrets.json is refused' + +echo "Security regression suite (operator-local only)" +echo "Repository policy: see .ship-policy.json" +echo "" + +bats --filter "$FILTER" test/ + +echo "" +echo "All filtered security regression tests passed." +echo "" + +if [ ! -t 0 ]; then + echo "Refusing non-interactive sign-off. Re-run in a terminal and complete operator certification." >&2 + exit 1 +fi + +read -r -p "Operator name: " OPERATOR +if [ -z "${OPERATOR//[[:space:]]/}" ]; then + echo "Operator name is required." >&2 + exit 1 +fi + +read -r -p "Type SIGNOFF to certify you ran this suite locally: " CONFIRM +if [ "$CONFIRM" != "SIGNOFF" ]; then + echo "Sign-off aborted (expected exactly SIGNOFF)." >&2 + exit 1 +fi + +SIGNOFF_DIR="$ROOT/.gstack" +mkdir -p "$SIGNOFF_DIR" +COMMIT="$(git rev-parse HEAD 2>/dev/null || echo unknown)" +TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +SIGNOFF_FILE="$SIGNOFF_DIR/security-signoff.json" + +# Escape operator name for JSON (minimal — names should not contain quotes). +OPERATOR_JSON="${OPERATOR//\\/\\\\}" +OPERATOR_JSON="${OPERATOR_JSON//\"/\\\"}" + +cat >"$SIGNOFF_FILE" < Date: Sun, 7 Jun 2026 13:50:51 -0700 Subject: [PATCH 11/12] chore: bump version and changelog (v0.4.0.0) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e6e27..d7f6ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.4.0.0] - 2026-06-07 + +### Added + +- **`.secrets.json` manifest (EGB-677 stage 1)** — a committed, project-root + manifest is now the source of truth for what syncs. List the env files you + want under `dotenv[]` (project-relative, nested paths and `@`-scoped + workspaces allowed; `..`, absolute, and symlink paths are rejected) and + out-of-project files under `external[]` (`properties` or `file`). The + manifest is shared across machines, so a teammate who clones the project + sees exactly what to pull. +- **`secrets add `** — declare an env file in the manifest without + pushing. Bootstraps `.secrets.json` on first use, dedupes, and writes a + stable canonical form. +- **Auto-add on push** — `secrets push` discovers new `.env*` / `.dev.vars` + files and adds them to the manifest (prints what it added and reminds you to + commit). Gated by `options.autoAdd` in the manifest (default on); + `push --frozen` syncs only declared files, and `push --dry-run` previews + what would change without writing anything. +- **Manifest-driven pull** — restores every declared file, recreating nested + directories as needed, with the same path-safety rail applied at restore + time so a malicious committed manifest can't write outside the project. An + empty manifest is a safe no-op. +- **Legacy `.secrets-files` absorb** — an existing `.secrets-files` is folded + into `.secrets.json` on first push (gradle-properties entries become + `properties`); on pull the legacy file is superseded with a warning. +- **Platform-aware install hints** — missing-dependency errors now print the + right install command for your platform (brew / apt-get / dnf). + +### Changed + +- `jq` is required only when a manifest is present or being written; + manifest-less projects keep working without `jq` (manifest features are + skipped with a notice). + +### Fixed + +- **Key rotation no longer orphans nested or external blobs.** `secrets rekey` + and `secrets list` now walk the entire project tree, so nested manifest + entries (`/.age`) and `external/` blobs are re-encrypted + and listed correctly. Previously a rekey could leave nested blobs encrypted + under the discarded old key, making them permanently undecryptable. +- Test assertions now fail correctly under system bash 3.2 (standalone + `[[ ]]` checks no longer pass silently). + ## [0.3.0.0] - 2026-06-07 ### Added diff --git a/VERSION b/VERSION index 1da00ae..9551b0d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0.0 +0.4.0.0 From a35bff8d4b214c99e3225dd7770ee6bda7e98885 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 13:53:05 -0700 Subject: [PATCH 12/12] docs: correct manifest.bats and total test counts for v0.4.0.0 CLAUDE.md: manifest.bats 58 -> 60 tests (actual @test count). README.md: total 191 -> 193 tests across both files. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 174bf07..262818d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,7 +74,7 @@ secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ secrets.bats # bats-core test suite (133 tests) - manifest.bats # EGB-677 .secrets.json manifest tests (58 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (60 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index 421aad5..770a86d 100644 --- a/README.md +++ b/README.md @@ -494,7 +494,7 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` ## Development ```bash -# Run the test suite (191 tests across both files) +# Run the test suite (193 tests across both files) brew install bats-core bats test/