feat: secrets verify — manifest↔store consistency + decrypt integrity (EGB-698)
Read-only integrity check, the safety net for the stage-2 store migration. Default mode (current project) cross-checks $PWD/.secrets.json against the store both ways — declared-but-missing blobs and orphaned blobs (no manifest entry) — and decrypt-tests every dotenv + external blob with the current key, streaming plaintext to /dev/null so nothing is ever written to disk. `verify --all` decrypt-tests every blob in every project (integrity only; the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (find -type f), the same walk rekey/list use, so nested manifest blobs are covered. Exits non-zero on any finding so it can gate `migrate --finalize` and CI. 12 bats tests (clean, nested+external, missing blob, decrypt failure, orphan, missing external, no-manifest die, symlink refusal, --all clean/corrupt/orphan, nested decrypt failure). Full suite 205/205. bash 3.2 clean.
This commit is contained in:
Brian Majewski
parent
fb71b956da
commit
52528f2e06
4 changed files with 291 additions and 1 deletions
|
|
@ -56,12 +56,13 @@ skips security specialist + red team, and Step 11 skips adversarial review.
|
|||
|
||||
## Architecture
|
||||
|
||||
Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey.
|
||||
Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey, verify.
|
||||
|
||||
- Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable)
|
||||
- Storage: Private git repo at `~/.secrets/`
|
||||
- Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`)
|
||||
- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `<project>/<relpath>.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints.
|
||||
- Verify (EGB-698): `secrets verify` is a read-only integrity check. Default mode (current project) cross-checks `$PWD/.secrets.json` against `$SECRETS_DIR/<project>/` both ways (declared-but-missing blobs + orphaned blobs) and decrypt-tests every blob (dotenv + external) by streaming plaintext to `/dev/null` (never written to disk). `secrets verify --all` decrypt-tests every blob in every project (integrity only — the store carries no manifests, so consistency can't be checked store-wide). Both recurse the whole project tree (`find -type f`, same as rekey/list). Exits non-zero on any finding so it can gate the stage-2 `migrate --finalize` and CI. The store deliberately holds no manifest — `.secrets.json` is committed in each project's own repo and read from `$PWD`.
|
||||
- External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below
|
||||
- Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq`
|
||||
- Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`)
|
||||
|
|
|
|||
|
|
@ -169,6 +169,8 @@ secrets clear
|
|||
| `secrets list` | Show all projects that have stored secrets |
|
||||
| `secrets rm <project>` | Delete a project's secrets from the store |
|
||||
| `secrets rekey` | Generate a new encryption key and re-encrypt everything |
|
||||
| `secrets verify` | Check the current project's `.secrets.json` against the store (missing/orphaned blobs) and decrypt every blob |
|
||||
| `secrets verify --all` | Decrypt-test every blob in every project — a store-wide integrity sweep |
|
||||
|
||||
### Automatic project detection
|
||||
|
||||
|
|
|
|||
158
secrets
158
secrets
|
|
@ -1907,6 +1907,161 @@ cmd_which() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Decrypt-test one blob with the current key. Plaintext is streamed to
|
||||
# /dev/null and never written to disk (read-only contract). Returns 0 if the
|
||||
# blob decrypts, non-zero otherwise.
|
||||
_verify_blob_decrypts() {
|
||||
age -d -i "$KEY_FILE" "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# `secrets verify --all` — store-wide decrypt sweep. Decrypt-tests every blob
|
||||
# in every project. No manifest consistency check: the store carries only
|
||||
# ciphertext (manifests live in each project's repo), so orphan/missing
|
||||
# detection is impossible store-wide. This is the migration integrity gate.
|
||||
_verify_all() {
|
||||
local checked=0 failed=0 dir project f rel
|
||||
for dir in "$SECRETS_DIR"/*/; do
|
||||
[ -d "$dir" ] || continue
|
||||
project=$(basename "$dir")
|
||||
[[ "$project" == .* ]] && continue
|
||||
# Recurse the whole project tree (top-level / nested / external) — never a
|
||||
# non-recursive glob, or nested blobs would be silently skipped (the exact
|
||||
# class of bug that orphaned nested blobs on rekey before v0.4.0.0).
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
checked=$((checked + 1))
|
||||
if ! _verify_blob_decrypts "$f"; then
|
||||
rel="${f#"$SECRETS_DIR"/}"
|
||||
echo "FAIL: $rel does not decrypt with the current key." >&2
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done < <(find "$dir" -type f -name '*.age')
|
||||
done
|
||||
if [ "$checked" -eq 0 ]; then
|
||||
info "verify --all: store is empty — nothing to check."
|
||||
return 0
|
||||
fi
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
echo "verify --all: $failed of $checked blob(s) failed to decrypt." >&2
|
||||
return 1
|
||||
fi
|
||||
info "verify --all: OK — all $checked blob(s) decrypt with the current key (integrity only; run 'secrets verify' in a project for manifest consistency)."
|
||||
}
|
||||
|
||||
# `secrets verify [project]` — current-project consistency + decrypt check.
|
||||
# Cross-checks the committed .secrets.json against the store both ways
|
||||
# (declared-but-missing blobs, orphaned blobs) and decrypt-tests every blob.
|
||||
_verify_project() {
|
||||
local explicit_project="$1"
|
||||
local manifest="$PWD/$SECRETS_JSON_NAME"
|
||||
if [ ! -e "$manifest" ]; then
|
||||
die "No $SECRETS_JSON_NAME in $PWD.
|
||||
'secrets verify' checks a project's manifest against the store. Either cd
|
||||
into a project that has a $SECRETS_JSON_NAME, or run 'secrets verify --all'
|
||||
for a store-wide decrypt sweep."
|
||||
fi
|
||||
_check_manifest_file "$manifest" # fatal on symlink / malformed / version
|
||||
|
||||
local project
|
||||
project=$(derive_project_name "$explicit_project")
|
||||
local pdir="$SECRETS_DIR/$project"
|
||||
|
||||
local findings=0 checked=0
|
||||
# `expected` accumulates the store-relative blob paths the manifest implies,
|
||||
# newline-framed (leading + trailing \n per entry) so the orphan walk can
|
||||
# test membership. bash 3.2 has no associative arrays — this string-set +
|
||||
# case match mirrors the `seen` pattern in _external_entries_for_push.
|
||||
local expected=$'\n'
|
||||
|
||||
# ── dotenv entries: rail + missing-blob + decrypt ──
|
||||
local rel blob
|
||||
while IFS= read -r rel; do
|
||||
[ -n "$rel" ] || continue
|
||||
if ! _validate_dotenv_rel_path "$rel" 2>/dev/null; then
|
||||
echo "FINDING: unsafe dotenv path in $SECRETS_JSON_NAME: '$rel' (will be refused)." >&2
|
||||
findings=$((findings + 1))
|
||||
continue
|
||||
fi
|
||||
expected="$expected$rel.age"$'\n'
|
||||
blob="$pdir/$rel.age"
|
||||
if [ ! -f "$blob" ]; then
|
||||
echo "FINDING: '$rel' is declared in $SECRETS_JSON_NAME but has no blob in the store ($project/$rel.age missing). Run 'secrets push'." >&2
|
||||
findings=$((findings + 1))
|
||||
continue
|
||||
fi
|
||||
checked=$((checked + 1))
|
||||
if ! _verify_blob_decrypts "$blob"; then
|
||||
echo "FINDING: blob for '$rel' ($project/$rel.age) does not decrypt with the current key." >&2
|
||||
findings=$((findings + 1))
|
||||
fi
|
||||
done < <(jq -r '.dotenv // [] | .[]' "$manifest")
|
||||
|
||||
# ── external entries: missing-blob + decrypt ──
|
||||
local etype epath ekeys slug erel eblob
|
||||
while IFS=$'\t' read -r etype epath ekeys; do
|
||||
[ -n "$etype" ] || continue
|
||||
slug=$(_secrets_files_slug "$epath")
|
||||
erel="external/$slug.$etype.age"
|
||||
expected="$expected$erel"$'\n'
|
||||
eblob="$pdir/$erel"
|
||||
if [ ! -f "$eblob" ]; then
|
||||
echo "FINDING: external '$epath' ($etype) is declared but has no blob in the store ($project/$erel missing). Run 'secrets push'." >&2
|
||||
findings=$((findings + 1))
|
||||
continue
|
||||
fi
|
||||
checked=$((checked + 1))
|
||||
if ! _verify_blob_decrypts "$eblob"; then
|
||||
echo "FINDING: external blob for '$epath' ($project/$erel) does not decrypt with the current key." >&2
|
||||
findings=$((findings + 1))
|
||||
fi
|
||||
done < <(_json_external_entries "$manifest")
|
||||
|
||||
# ── orphan detection: any stored blob the manifest doesn't account for ──
|
||||
if [ -d "$pdir" ]; then
|
||||
local f frel
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
frel="${f#"$pdir"/}"
|
||||
case "$expected" in
|
||||
*$'\n'"$frel"$'\n'*) : ;; # accounted for by a manifest entry
|
||||
*)
|
||||
echo "FINDING: orphan blob $project/$frel has no entry in $SECRETS_JSON_NAME." >&2
|
||||
findings=$((findings + 1))
|
||||
;;
|
||||
esac
|
||||
done < <(find "$pdir" -type f -name '*.age')
|
||||
fi
|
||||
|
||||
if [ "$findings" -gt 0 ]; then
|
||||
echo "verify: $findings finding(s) for project '$project'." >&2
|
||||
return 1
|
||||
fi
|
||||
info "verify: OK — $checked blob(s) verified for '$project' (manifest and store agree; all decrypt)."
|
||||
}
|
||||
|
||||
# `secrets verify [--all] [project]` — read-only integrity check. Exits
|
||||
# non-zero on any finding so it can gate `migrate --finalize` and CI.
|
||||
cmd_verify() {
|
||||
resolve_store
|
||||
check_initialized
|
||||
check_key
|
||||
|
||||
local all=false explicit_project=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--all) all=true; shift ;;
|
||||
-*) die "Unknown verify flag: $1. Usage: secrets verify [--all] [project]" ;;
|
||||
*) explicit_project="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$all" = true ]; then
|
||||
_verify_all
|
||||
else
|
||||
_verify_project "$explicit_project"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_help() {
|
||||
cat << 'EOF'
|
||||
secrets — encrypted secret file sync between machines
|
||||
|
|
@ -1926,6 +2081,8 @@ Usage:
|
|||
secrets list List all projects and their secret files
|
||||
secrets rm <project> Remove a project's secrets from the repo
|
||||
secrets rekey Re-encrypt all secrets with a new key
|
||||
secrets verify [project] Check the manifest against the store + decrypt every blob
|
||||
secrets verify --all Decrypt-test every blob in every project (integrity gate)
|
||||
secrets which Show the active store, manifest, and external entries
|
||||
secrets where Alias for `which`
|
||||
secrets status Alias for `which`
|
||||
|
|
@ -2094,6 +2251,7 @@ case "${1:-help}" in
|
|||
list) cmd_list ;;
|
||||
rm) cmd_rm "${2:-}" ;;
|
||||
rekey) cmd_rekey ;;
|
||||
verify) shift; cmd_verify "$@" ;;
|
||||
which|where|status) cmd_which ;;
|
||||
help|--help|-h) cmd_help ;;
|
||||
*) die "Unknown command: $1. Run 'secrets help' for usage." ;;
|
||||
|
|
|
|||
|
|
@ -731,3 +731,132 @@ m_nojq_path() {
|
|||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"packages/web/.env.development"* ]] || false
|
||||
}
|
||||
|
||||
# ─── K: secrets verify — manifest↔store consistency + decrypt integrity (EGB-698) ─
|
||||
|
||||
@test "verify: clean pushed project reports OK and exits 0" {
|
||||
init_with_remote
|
||||
create_project_dir verifyok
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"OK"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify: nested + external entries all pass" {
|
||||
init_with_remote
|
||||
m_gradle_src $'beaconClerkPkTest=pk_test_abc\n'
|
||||
create_project_dir verifymix
|
||||
mkdir -p packages/web
|
||||
echo "N=nested" > packages/web/.env.development
|
||||
"$SECRETS_BIN" add packages/web/.env.development >/dev/null
|
||||
printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "verify: declared-but-missing blob is a finding (exit 1)" {
|
||||
init_with_remote
|
||||
create_project_dir verifymiss
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
rm "$SECRETS_DIR/verifymiss/.env.staging.age"
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *".env.staging"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify: a blob that fails to decrypt is a finding (exit 1)" {
|
||||
init_with_remote
|
||||
create_project_dir verifycorrupt
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
printf 'not-a-valid-age-blob' > "$SECRETS_DIR/verifycorrupt/.env.age"
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *".env"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify: orphan blob (no manifest entry) is a finding (exit 1)" {
|
||||
init_with_remote
|
||||
create_project_dir verifyorphan
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
# A valid, decryptable blob with no manifest entry — pure consistency miss.
|
||||
cp "$SECRETS_DIR/verifyorphan/.env.age" "$SECRETS_DIR/verifyorphan/.stray.age"
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"stray"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify: missing external blob is a finding (exit 1)" {
|
||||
init_with_remote
|
||||
m_gradle_src $'beaconClerkPkTest=pk_test_abc\n'
|
||||
create_project_dir verifyextmiss
|
||||
printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\n' > .secrets-files
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
rm "$SECRETS_DIR/verifyextmiss/external/"*.age
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "verify: no manifest in cwd dies with a directed message" {
|
||||
init_with_remote
|
||||
local dir="$WORK_DIR/verifynomani"; mkdir -p "$dir"; cd "$dir"
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *".secrets.json"* ]] || false
|
||||
[[ "$output" == *"--all"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify: symlinked manifest is refused" {
|
||||
init_with_remote
|
||||
create_project_dir verifysymlink
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
rm .secrets.json
|
||||
ln -s /etc/hosts .secrets.json
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"symlink"* ]] || false
|
||||
}
|
||||
|
||||
@test "verify --all: clean store passes (decrypt-only)" {
|
||||
init_with_remote
|
||||
create_project_dir verifyall1
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
cd "$WORK_DIR"
|
||||
run "$SECRETS_BIN" verify --all
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "verify --all: a corrupted blob anywhere fails (exit 1)" {
|
||||
init_with_remote
|
||||
create_project_dir verifyall2
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
printf 'garbage' > "$SECRETS_DIR/verifyall2/.env.age"
|
||||
cd "$WORK_DIR"
|
||||
run "$SECRETS_BIN" verify --all
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "verify --all: an orphan that decrypts is NOT flagged (no consistency check)" {
|
||||
init_with_remote
|
||||
create_project_dir verifyall3
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
# Orphan blob that decrypts fine — default mode flags it, --all does not.
|
||||
cp "$SECRETS_DIR/verifyall3/.env.age" "$SECRETS_DIR/verifyall3/.stray.age"
|
||||
cd "$WORK_DIR"
|
||||
run "$SECRETS_BIN" verify --all
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "verify: nested blob that fails to decrypt is caught (rekey-orphan guard)" {
|
||||
init_with_remote
|
||||
create_project_dir verifynested
|
||||
mkdir -p packages/web
|
||||
echo "N=nested" > packages/web/.env.development
|
||||
"$SECRETS_BIN" add packages/web/.env.development >/dev/null
|
||||
"$SECRETS_BIN" push >/dev/null 2>&1
|
||||
printf 'broken' > "$SECRETS_DIR/verifynested/packages/web/.env.development.age"
|
||||
run "$SECRETS_BIN" verify
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"packages/web/.env.development"* ]] || false
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue