diff --git a/CLAUDE.md b/CLAUDE.md index 262818d..296e205 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,12 +56,13 @@ skips security specialist + red team, and Step 11 skips adversarial review. ## Architecture -Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. +Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify. - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) - Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. +- Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR//` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` - Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`) diff --git a/README.md b/README.md index 770a86d..fc6c1c7 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,8 @@ secrets clear | `secrets list` | Show all projects that have stored secrets | | `secrets rm ` | Delete a project's secrets from the store | | `secrets rekey` | Generate a new encryption key and re-encrypt everything | +| `secrets verify` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob | +| `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep | ### Automatic project detection diff --git a/secrets b/secrets index ba2adce..61ea874 100755 --- a/secrets +++ b/secrets @@ -1907,6 +1907,161 @@ cmd_which() { fi } +# Decrypt-test one blob with the current key. Plaintext is streamed to +# /dev/null and never written to disk (read-only contract). Returns 0 if the +# blob decrypts, non-zero otherwise. +_verify_blob_decrypts() { + age -d -i "$KEY_FILE" "$1" >/dev/null 2>&1 +} + +# `secrets verify --all` — store-wide decrypt sweep. Decrypt-tests every blob +# in every project. No manifest consistency check: the store carries only +# ciphertext (manifests live in each project's repo), so orphan/missing +# detection is impossible store-wide. This is the migration integrity gate. +_verify_all() { + local checked=0 failed=0 dir project f rel + for dir in "$SECRETS_DIR"/*/; do + [ -d "$dir" ] || continue + project=$(basename "$dir") + [[ "$project" == .* ]] && continue + # Recurse the whole project tree (top-level / nested / external) — never a + # non-recursive glob, or nested blobs would be silently skipped (the exact + # class of bug that orphaned nested blobs on rekey before v0.4.0.0). + while IFS= read -r f; do + [ -f "$f" ] || continue + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$f"; then + rel="${f#"$SECRETS_DIR"/}" + echo "FAIL: $rel does not decrypt with the current key." >&2 + failed=$((failed + 1)) + fi + done < <(find "$dir" -type f -name '*.age') + done + if [ "$checked" -eq 0 ]; then + info "verify --all: store is empty — nothing to check." + return 0 + fi + if [ "$failed" -gt 0 ]; then + echo "verify --all: $failed of $checked blob(s) failed to decrypt." >&2 + return 1 + fi + info "verify --all: OK — all $checked blob(s) decrypt with the current key (integrity only; run 'secrets verify' in a project for manifest consistency)." +} + +# `secrets verify [project]` — current-project consistency + decrypt check. +# Cross-checks the committed .secrets.json against the store both ways +# (declared-but-missing blobs, orphaned blobs) and decrypt-tests every blob. +_verify_project() { + local explicit_project="$1" + local manifest="$PWD/$SECRETS_JSON_NAME" + if [ ! -e "$manifest" ]; then + die "No $SECRETS_JSON_NAME in $PWD. + 'secrets verify' checks a project's manifest against the store. Either cd + into a project that has a $SECRETS_JSON_NAME, or run 'secrets verify --all' + for a store-wide decrypt sweep." + fi + _check_manifest_file "$manifest" # fatal on symlink / malformed / version + + local project + project=$(derive_project_name "$explicit_project") + local pdir="$SECRETS_DIR/$project" + + local findings=0 checked=0 + # `expected` accumulates the store-relative blob paths the manifest implies, + # newline-framed (leading + trailing \n per entry) so the orphan walk can + # test membership. bash 3.2 has no associative arrays — this string-set + + # case match mirrors the `seen` pattern in _external_entries_for_push. + local expected=$'\n' + + # ── dotenv entries: rail + missing-blob + decrypt ── + local rel blob + while IFS= read -r rel; do + [ -n "$rel" ] || continue + if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then + echo "FINDING: unsafe dotenv path in $SECRETS_JSON_NAME: '$rel' (will be refused)." >&2 + findings=$((findings + 1)) + continue + fi + expected="$expected$rel.age"$'\n' + blob="$pdir/$rel.age" + if [ ! -f "$blob" ]; then + echo "FINDING: '$rel' is declared in $SECRETS_JSON_NAME but has no blob in the store ($project/$rel.age missing). Run 'secrets push'." >&2 + findings=$((findings + 1)) + continue + fi + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$blob"; then + echo "FINDING: blob for '$rel' ($project/$rel.age) does not decrypt with the current key." >&2 + findings=$((findings + 1)) + fi + done < <(jq -r '.dotenv // [] | .[]' "$manifest") + + # ── external entries: missing-blob + decrypt ── + local etype epath ekeys slug erel eblob + while IFS=$'\t' read -r etype epath ekeys; do + [ -n "$etype" ] || continue + slug=$(_secrets_files_slug "$epath") + erel="external/$slug.$etype.age" + expected="$expected$erel"$'\n' + eblob="$pdir/$erel" + if [ ! -f "$eblob" ]; then + echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2 + findings=$((findings + 1)) + continue + fi + checked=$((checked + 1)) + if ! _verify_blob_decrypts "$eblob"; then + echo "FINDING: external blob for '$epath' ($project/$erel) does not decrypt with the current key." >&2 + findings=$((findings + 1)) + fi + done < <(_json_external_entries "$manifest") + + # ── orphan detection: any stored blob the manifest doesn't account for ── + if [ -d "$pdir" ]; then + local f frel + while IFS= read -r f; do + [ -f "$f" ] || continue + frel="${f#"$pdir"/}" + case "$expected" in + *$'\n'"$frel"$'\n'*) : ;; # accounted for by a manifest entry + *) + echo "FINDING: orphan blob $project/$frel has no entry in $SECRETS_JSON_NAME." >&2 + findings=$((findings + 1)) + ;; + esac + done < <(find "$pdir" -type f -name '*.age') + fi + + if [ "$findings" -gt 0 ]; then + echo "verify: $findings finding(s) for project '$project'." >&2 + return 1 + fi + info "verify: OK — $checked blob(s) verified for '$project' (manifest and store agree; all decrypt)." +} + +# `secrets verify [--all] [project]` — read-only integrity check. Exits +# non-zero on any finding so it can gate `migrate --finalize` and CI. +cmd_verify() { + resolve_store + check_initialized + check_key + + local all=false explicit_project="" + while [ $# -gt 0 ]; do + case "$1" in + --all) all=true; shift ;; + -*) die "Unknown verify flag: $1. Usage: secrets verify [--all] [project]" ;; + *) explicit_project="$1"; shift ;; + esac + done + + if [ "$all" = true ]; then + _verify_all + else + _verify_project "$explicit_project" + fi +} + cmd_help() { cat << 'EOF' secrets — encrypted secret file sync between machines @@ -1926,6 +2081,8 @@ Usage: secrets list List all projects and their secret files secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key + secrets verify [project] Check the manifest against the store + decrypt every blob + secrets verify --all Decrypt-test every blob in every project (integrity gate) secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` @@ -2094,6 +2251,7 @@ case "${1:-help}" in list) cmd_list ;; rm) cmd_rm "${2:-}" ;; rekey) cmd_rekey ;; + verify) shift; cmd_verify "$@" ;; which|where|status) cmd_which ;; help|--help|-h) cmd_help ;; *) die "Unknown command: $1. Run 'secrets help' for usage." ;; diff --git a/test/manifest.bats b/test/manifest.bats index 4459a6c..2a226d1 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -731,3 +731,132 @@ m_nojq_path() { [ "$status" -eq 0 ] [[ "$output" == *"packages/web/.env.development"* ]] || false } + +# ─── K: secrets verify — manifest↔store consistency + decrypt integrity (EGB-698) ─ + +@test "verify: clean pushed project reports OK and exits 0" { + init_with_remote + create_project_dir verifyok + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" verify + [ "$status" -eq 0 ] + [[ "$output" == *"OK"* ]] || false +} + +@test "verify: nested + external entries all pass" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir verifymix + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + run "$SECRETS_BIN" verify + [ "$status" -eq 0 ] +} + +@test "verify: declared-but-missing blob is a finding (exit 1)" { + init_with_remote + create_project_dir verifymiss + "$SECRETS_BIN" push >/dev/null 2>&1 + rm "$SECRETS_DIR/verifymiss/.env.staging.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".env.staging"* ]] || false +} + +@test "verify: a blob that fails to decrypt is a finding (exit 1)" { + init_with_remote + create_project_dir verifycorrupt + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'not-a-valid-age-blob' > "$SECRETS_DIR/verifycorrupt/.env.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".env"* ]] || false +} + +@test "verify: orphan blob (no manifest entry) is a finding (exit 1)" { + init_with_remote + create_project_dir verifyorphan + "$SECRETS_BIN" push >/dev/null 2>&1 + # A valid, decryptable blob with no manifest entry — pure consistency miss. + cp "$SECRETS_DIR/verifyorphan/.env.age" "$SECRETS_DIR/verifyorphan/.stray.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"stray"* ]] || false +} + +@test "verify: missing external blob is a finding (exit 1)" { + init_with_remote + m_gradle_src $'beaconClerkPkTest=pk_test_abc\n' + create_project_dir verifyextmiss + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files + "$SECRETS_BIN" push >/dev/null 2>&1 + rm "$SECRETS_DIR/verifyextmiss/external/"*.age + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] +} + +@test "verify: no manifest in cwd dies with a directed message" { + init_with_remote + local dir="$WORK_DIR/verifynomani"; mkdir -p "$dir"; cd "$dir" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *".secrets.json"* ]] || false + [[ "$output" == *"--all"* ]] || false +} + +@test "verify: symlinked manifest is refused" { + init_with_remote + create_project_dir verifysymlink + "$SECRETS_BIN" push >/dev/null 2>&1 + rm .secrets.json + ln -s /etc/hosts .secrets.json + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"symlink"* ]] || false +} + +@test "verify --all: clean store passes (decrypt-only)" { + init_with_remote + create_project_dir verifyall1 + "$SECRETS_BIN" push >/dev/null 2>&1 + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 0 ] +} + +@test "verify --all: a corrupted blob anywhere fails (exit 1)" { + init_with_remote + create_project_dir verifyall2 + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'garbage' > "$SECRETS_DIR/verifyall2/.env.age" + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 1 ] +} + +@test "verify --all: an orphan that decrypts is NOT flagged (no consistency check)" { + init_with_remote + create_project_dir verifyall3 + "$SECRETS_BIN" push >/dev/null 2>&1 + # Orphan blob that decrypts fine — default mode flags it, --all does not. + cp "$SECRETS_DIR/verifyall3/.env.age" "$SECRETS_DIR/verifyall3/.stray.age" + cd "$WORK_DIR" + run "$SECRETS_BIN" verify --all + [ "$status" -eq 0 ] +} + +@test "verify: nested blob that fails to decrypt is caught (rekey-orphan guard)" { + init_with_remote + create_project_dir verifynested + mkdir -p packages/web + echo "N=nested" > packages/web/.env.development + "$SECRETS_BIN" add packages/web/.env.development >/dev/null + "$SECRETS_BIN" push >/dev/null 2>&1 + printf 'broken' > "$SECRETS_DIR/verifynested/packages/web/.env.development.age" + run "$SECRETS_BIN" verify + [ "$status" -eq 1 ] + [[ "$output" == *"packages/web/.env.development"* ]] || false +}