feat: store-format-v2 self-describing migration (EGB-703, folds in EGB-700)
Stage 2 of EGB-677. Makes the store self-describing and unifies the legacy external `properties` blob suffix, via a copy-forward migration that never destroys data until an explicit, gated finalize. Scope decision (see eureka): the EGB-677 CEO plan's "flatten dotenv blobs to basename" was dropped as LOSSY — it discards the restore relpath that makes the store self-describing and adds basename collisions. Engineering analysis (4 parallel design agents) showed the store is already relpath-self-describing; the only real v1→v2 delta is the `properties` suffix. This implements the minimal, safe v2 that achieves the epic's self-describing goal. What's added: - `.secrets-format` marker (committed, one line `2`). Absence ⇒ v1 (every pre-EGB-703 store). `_store_format()` reads it; `init` stamps fresh stores born-v2. `secrets which` prints `format: vN` (EGB-700 folded in). - `_external_blob_suffix(type)` — single source of truth for the external suffix (v2: gradle-properties → properties; file unchanged). push/pull/verify all route through it, so v1 and v2 stores never disagree on blob location. - `secrets migrate` — per-project copy-forward (writes `.properties.age` twins beside v1 blobs; idempotent; needs the project manifest), `--dry-run` (reports old→new, writes nothing), `--finalize` (store-wide, the only destructive step: gates on `verify --all` green + every v1 blob twinned, cuts a `pre-v2-migrate-<sha>` recovery tag, stamps the marker, then drops v1 blobs; refuses without `--yes`/operator confirmation). rekey and verify --all stay format-agnostic (recursive find walk) — no change. 21 new bats tests (test/migrate.bats): marker/born-v2, format-aware suffix, v1 back-compat, dry-run, copy-forward idempotency, no-manifest die, finalize gates (verify-not-green refusal, untwinned refusal, recovery tag, confirmation), and full v1→window→finalize round-trip. Updated 4 existing tests for the born-v2 suffix. Full suite 231/231, bash 3.2 clean.
This commit is contained in:
Brian Majewski
parent
144ff3692b
commit
e2ad661da5
6 changed files with 487 additions and 10 deletions
212
secrets
212
secrets
|
|
@ -517,6 +517,39 @@ _secrets_files_slug() {
|
|||
printf '%s-%s' "$clean" "$sum"
|
||||
}
|
||||
|
||||
# ─── Store format (EGB-703) ───────────────────────────────────────────
|
||||
#
|
||||
# The store self-describes its format via a committed one-line file
|
||||
# `$SECRETS_DIR/.secrets-format` containing `2`. Absence (or any non-`2`
|
||||
# content) means format v1 — the legacy default for every store that
|
||||
# predates EGB-703. `init` stamps a fresh store v2 (born-v2); `migrate
|
||||
# --finalize` stamps a migrated store v2. Requires resolve_store to have
|
||||
# run (SECRETS_DIR set).
|
||||
STORE_FORMAT_FILE_NAME=".secrets-format"
|
||||
_store_format() {
|
||||
local f="$SECRETS_DIR/$STORE_FORMAT_FILE_NAME" v
|
||||
if [ -f "$f" ]; then
|
||||
v=$(head -1 "$f" 2>/dev/null | tr -dc '0-9')
|
||||
[ "$v" = "2" ] && { echo 2; return; }
|
||||
fi
|
||||
echo 1
|
||||
}
|
||||
|
||||
# The on-disk blob suffix for an external entry, format-aware. v2 unifies
|
||||
# the legacy `gradle-properties` suffix to `properties` (matching the JSON
|
||||
# manifest `type`); `file` is unchanged in both formats. The slug + this
|
||||
# suffix + `.age` is the external blob name. This is the single source of
|
||||
# truth for the suffix — push, pull, verify all route through it so a v1
|
||||
# and a v2 store can never disagree on where a blob lives.
|
||||
_external_blob_suffix() {
|
||||
local mtype="$1"
|
||||
if [ "$mtype" = "gradle-properties" ] && [ "$(_store_format)" = "2" ]; then
|
||||
echo "properties"
|
||||
else
|
||||
echo "$mtype"
|
||||
fi
|
||||
}
|
||||
|
||||
# Merge managed key=value lines (from $2) into target file $1, preserving
|
||||
# all unrelated lines/comments/order. Updates a managed key in place (first
|
||||
# occurrence), collapses duplicates, appends new keys. Atomic + mode-safe.
|
||||
|
|
@ -615,7 +648,7 @@ push_external_files() {
|
|||
# EGB-652: whole-file sync — encrypt the file verbatim (binary-safe).
|
||||
mkdir -p "$SECRETS_DIR/$project/external"
|
||||
local fslug; fslug=$(_secrets_files_slug "$mpath")
|
||||
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.file.age" "$expanded"
|
||||
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.$(_external_blob_suffix file).age" "$expanded"
|
||||
info "Encrypted file $mpath"
|
||||
pushed=$((pushed + 1))
|
||||
continue
|
||||
|
|
@ -644,7 +677,7 @@ push_external_files() {
|
|||
fi
|
||||
mkdir -p "$SECRETS_DIR/$project/external"
|
||||
local slug; slug=$(_secrets_files_slug "$mpath")
|
||||
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$mtype.age" "$tmp"
|
||||
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age" "$tmp"
|
||||
rm -f "$tmp"
|
||||
info "Extracted $found key(s) from $mpath"
|
||||
pushed=$((pushed + 1))
|
||||
|
|
@ -673,7 +706,7 @@ pull_external_files() {
|
|||
continue
|
||||
fi
|
||||
local slug; slug=$(_secrets_files_slug "$mpath")
|
||||
local blob="$SECRETS_DIR/$project/external/$slug.$mtype.age"
|
||||
local blob="$SECRETS_DIR/$project/external/$slug.$(_external_blob_suffix "$mtype").age"
|
||||
if [ ! -f "$blob" ]; then
|
||||
echo "WARNING: $SECRETS_FILES_NAME names '$mpath' but no encrypted data exists in the store yet. Run 'secrets push' on a machine that has these keys. Skipping." >&2
|
||||
continue
|
||||
|
|
@ -1126,6 +1159,11 @@ Your key file has been left untouched."
|
|||
# Write .gitignore
|
||||
write_store_gitignore
|
||||
|
||||
# Stamp the store format (EGB-703): a fresh store is born v2 — it has no
|
||||
# v1 blobs, so it is already in v2 shape. The marker is a committed,
|
||||
# non-secret metadata file (NOT gitignored); the first push stages it.
|
||||
printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME"
|
||||
|
||||
# Install pre-commit hook
|
||||
mkdir -p "$SECRETS_DIR/.git/hooks"
|
||||
install_hook
|
||||
|
|
@ -1866,6 +1904,9 @@ cmd_which() {
|
|||
resolve_store
|
||||
echo "store: $SECRETS_DIR"
|
||||
echo "source: $STORE_SOURCE"
|
||||
# EGB-700: surface the store format so users can tell v1 from v2 during the
|
||||
# migration window. A v1 store is a legacy store with no format marker.
|
||||
echo "format: v$(_store_format)"
|
||||
|
||||
# v2 manifest (.secrets.json): validate and summarize. Validation here
|
||||
# is deliberately fatal (symlink / malformed / unsupported version) so
|
||||
|
|
@ -2005,7 +2046,7 @@ _verify_project() {
|
|||
while IFS=$'\t' read -r etype epath _; do
|
||||
[ -n "$etype" ] || continue
|
||||
slug=$(_secrets_files_slug "$epath")
|
||||
erel="external/$slug.$etype.age"
|
||||
erel="external/$slug.$(_external_blob_suffix "$etype").age"
|
||||
expected="$expected$erel"$'\n'
|
||||
eblob="$pdir/$erel"
|
||||
if [ ! -f "$eblob" ]; then
|
||||
|
|
@ -2066,6 +2107,166 @@ cmd_verify() {
|
|||
fi
|
||||
}
|
||||
|
||||
# ─── Store-format-v2 migration (EGB-703) ──────────────────────────────
|
||||
#
|
||||
# v2 renames the legacy `properties` blob suffix (.gradle-properties.age →
|
||||
# .properties.age) and marks the store self-describing via .secrets-format.
|
||||
# Migration is copy-forward and non-destructive until --finalize:
|
||||
# secrets migrate --dry-run # per project: report old→new, write nothing
|
||||
# secrets migrate # per project: write v2 twins beside v1 blobs
|
||||
# secrets migrate --finalize # store-wide: verify, drop v1, stamp v2
|
||||
# Per-project (needs the project manifest to know which externals are
|
||||
# `properties`); finalize is store-wide. Mirrors verify's project/--all split.
|
||||
|
||||
# Copy-forward (or dry-run preview) for the current project. Reads
|
||||
# $PWD/.secrets.json; only `properties` external blobs rename in v2.
|
||||
_migrate_project() {
|
||||
local dry_run="$1"
|
||||
if [ "$(_store_format)" = "2" ]; then
|
||||
info "Store is already format v2 — nothing to migrate."
|
||||
return 0
|
||||
fi
|
||||
local manifest="$PWD/$SECRETS_JSON_NAME"
|
||||
if [ ! -e "$manifest" ]; then
|
||||
die "No $SECRETS_JSON_NAME in $PWD.
|
||||
'secrets migrate' copy-forwards a project's v1 blobs to their v2 names and
|
||||
reads the project manifest to do so. cd into a project that has a manifest,
|
||||
then run 'secrets migrate'. (Store-wide 'secrets migrate --finalize' comes
|
||||
after every project is migrated.)"
|
||||
fi
|
||||
_check_manifest_file "$manifest"
|
||||
local project; project=$(derive_project_name "")
|
||||
local pdir="$SECRETS_DIR/$project"
|
||||
|
||||
local moved=0 already=0 would=0 etype epath slug old new
|
||||
while IFS=$'\t' read -r etype epath _; do
|
||||
[ -n "$etype" ] || continue
|
||||
# Only `properties` blobs change name in v2; dotenv and `file` are already
|
||||
# in their v2 shape and never move.
|
||||
[ "$etype" = "gradle-properties" ] || continue
|
||||
slug=$(_secrets_files_slug "$epath")
|
||||
old="$pdir/external/$slug.gradle-properties.age"
|
||||
new="$pdir/external/$slug.properties.age"
|
||||
[ -f "$old" ] || continue # nothing pushed yet (or already dropped)
|
||||
if [ -f "$new" ]; then # idempotent: twin already exists
|
||||
already=$((already + 1))
|
||||
continue
|
||||
fi
|
||||
if [ "$dry_run" = true ]; then
|
||||
echo "would migrate: $project/external/$slug.gradle-properties.age -> $slug.properties.age"
|
||||
would=$((would + 1))
|
||||
else
|
||||
cp "$old" "$new"
|
||||
moved=$((moved + 1))
|
||||
fi
|
||||
done < <(_json_external_entries "$manifest")
|
||||
|
||||
if [ "$dry_run" = true ]; then
|
||||
echo "migrate --dry-run: $would blob(s) would be copy-forwarded for '$project' (writes nothing); $already already present. v1 blobs are kept until 'secrets migrate --finalize'."
|
||||
return 0
|
||||
fi
|
||||
if [ "$moved" -eq 0 ] && [ "$already" -eq 0 ]; then
|
||||
info "Nothing to migrate for '$project' (no v1 properties blobs)."
|
||||
return 0
|
||||
fi
|
||||
ensure_store_protections
|
||||
git -C "$SECRETS_DIR" add -A
|
||||
git -C "$SECRETS_DIR" commit -m "migrate: copy-forward v2 twins for $project" >/dev/null 2>&1 || true
|
||||
info "Copy-forward for '$project': $moved new v2 twin(s), $already already present. v1 blobs kept (non-destructive). Run 'secrets migrate --finalize' once every project is migrated and every machine is upgraded."
|
||||
}
|
||||
|
||||
# Store-wide finalize: the only destructive step. Refuses unless verify --all
|
||||
# is green and every v1 properties blob has a v2 twin. Cuts a recovery tag,
|
||||
# stamps the marker, then drops v1 blobs.
|
||||
_migrate_finalize() {
|
||||
local force="$1"
|
||||
check_key
|
||||
if [ "$(_store_format)" = "2" ]; then
|
||||
info "Store is already format v2 — nothing to finalize."
|
||||
return 0
|
||||
fi
|
||||
|
||||
# (gate 1) every blob must decrypt with the current key.
|
||||
info "Verifying every blob decrypts before finalizing..."
|
||||
if ! _verify_all >/dev/null 2>&1; then
|
||||
die "Refusing to finalize: 'secrets verify --all' is not green — a blob does not decrypt. Run 'secrets verify --all' to see which, fix it, then re-run --finalize."
|
||||
fi
|
||||
|
||||
# (gate 2) every v1 properties blob must have a v2 twin (project migrated).
|
||||
local untwinned="" f new v1count=0
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
v1count=$((v1count + 1))
|
||||
new="${f%.gradle-properties.age}.properties.age"
|
||||
[ -f "$new" ] || untwinned="$untwinned ${f#"$SECRETS_DIR"/}"$'\n'
|
||||
done < <(find "$SECRETS_DIR" -type f -name '*.gradle-properties.age')
|
||||
if [ -n "$untwinned" ]; then
|
||||
die "Refusing to finalize: these v1 properties blobs have no v2 twin (their project was not migrated):
|
||||
$untwinned cd into each project and run 'secrets migrate', then re-run 'secrets migrate --finalize'."
|
||||
fi
|
||||
if [ "$v1count" -eq 0 ]; then
|
||||
# No v1 blobs at all — just stamp the marker (dotenv/file-only store).
|
||||
printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME"
|
||||
ensure_store_protections
|
||||
git -C "$SECRETS_DIR" add -A
|
||||
git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2" >/dev/null 2>&1 || true
|
||||
git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true
|
||||
info "Store finalized to format v2 (no v1 blobs to drop)."
|
||||
return 0
|
||||
fi
|
||||
|
||||
# (gate 3) operator confirms every machine is upgraded.
|
||||
if [ "$force" != true ]; then
|
||||
echo "Finalize will drop $v1count v1 blob(s) and stamp the store format v2."
|
||||
echo "Any machine still running a pre-v2 'secrets' will stop seeing 'properties'"
|
||||
echo "externals from this store until it upgrades (git pull in the secrets repo)."
|
||||
printf "Confirm every machine is upgraded? Type 'yes': "
|
||||
local reply=""
|
||||
read -r reply < /dev/tty 2>/dev/null || read -r reply || true
|
||||
[ "$reply" = "yes" ] || die "Finalize aborted — no confirmation."
|
||||
fi
|
||||
|
||||
# Recovery tag BEFORE any mutation: points at the pre-finalize commit (v1
|
||||
# blobs intact, no marker), so `git checkout <tag>` fully restores v1.
|
||||
local tag="pre-v2-migrate-$(git -C "$SECRETS_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||
git -C "$SECRETS_DIR" tag "$tag" >/dev/null 2>&1 || true
|
||||
|
||||
# Stamp the marker FIRST, then drop v1 blobs. If finalize crashes between
|
||||
# the two, the store reads as v2 and the (verified) v2 twins serve every
|
||||
# upgraded client; leftover v1 blobs are harmless orphans a re-run cleans.
|
||||
printf '2\n' > "$SECRETS_DIR/$STORE_FORMAT_FILE_NAME"
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
rm -f "$f"
|
||||
done < <(find "$SECRETS_DIR" -type f -name '*.gradle-properties.age')
|
||||
|
||||
ensure_store_protections
|
||||
git -C "$SECRETS_DIR" add -A
|
||||
git -C "$SECRETS_DIR" commit -m "migrate: finalize store format v2 (drop $v1count v1 blob(s))" >/dev/null 2>&1 || true
|
||||
git -C "$SECRETS_DIR" remote get-url origin >/dev/null 2>&1 && git -C "$SECRETS_DIR" push >/dev/null 2>&1 || true
|
||||
info "Store finalized to format v2. Dropped $v1count v1 blob(s). Recovery tag in the store: $tag"
|
||||
}
|
||||
|
||||
cmd_migrate() {
|
||||
resolve_store
|
||||
check_initialized
|
||||
local dry_run=false finalize=false force=false
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dry-run) dry_run=true; shift ;;
|
||||
--finalize) finalize=true; shift ;;
|
||||
--yes|--force) force=true; shift ;;
|
||||
-*) die "Unknown migrate flag: $1. Usage: secrets migrate [--dry-run | --finalize] [--yes]" ;;
|
||||
*) die "migrate takes no project argument. Run it from inside a project (copy-forward) or use --finalize (store-wide)." ;;
|
||||
esac
|
||||
done
|
||||
if [ "$finalize" = true ]; then
|
||||
_migrate_finalize "$force"
|
||||
else
|
||||
_migrate_project "$dry_run"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_help() {
|
||||
cat << 'EOF'
|
||||
secrets — encrypted secret file sync between machines
|
||||
|
|
@ -2087,6 +2288,8 @@ Usage:
|
|||
secrets rekey Re-encrypt all secrets with a new key
|
||||
secrets verify [project] Check the manifest against the store + decrypt every blob
|
||||
secrets verify --all Decrypt-test every blob in every project (integrity gate)
|
||||
secrets migrate [--dry-run] Copy-forward this project's blobs to store format v2
|
||||
secrets migrate --finalize Drop v1 blobs and mark the store v2 (after verify)
|
||||
secrets which Show the active store, manifest, and external entries
|
||||
secrets where Alias for `which`
|
||||
secrets status Alias for `which`
|
||||
|
|
@ -2256,6 +2459,7 @@ case "${1:-help}" in
|
|||
rm) cmd_rm "${2:-}" ;;
|
||||
rekey) cmd_rekey ;;
|
||||
verify) shift; cmd_verify "$@" ;;
|
||||
migrate) shift; cmd_migrate "$@" ;;
|
||||
which|where|status) cmd_which ;;
|
||||
help|--help|-h) cmd_help ;;
|
||||
*) die "Unknown command: $1. Run 'secrets help' for usage." ;;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue