v0.2.0.0 feat: sync gradle.properties keys via .secrets-files (EGB-531)

Add a committed .secrets-files manifest that lets secrets track designated
keys from files outside the project root (motivating case:
~/.gradle/gradle.properties for Android Clerk publishable keys, which
Android Studio GUI builds read but terminal env vars can't reach).

- push extracts only the named keys, encrypts under <project>/external/
- pull MERGES them into the target, preserving unrelated keys/comments/order
- pure-bash merge (no sed/regex): exact-string key match, opaque values
- path validator: basename gradle.properties, within $HOME, no symlink/..
- external/ subdir keeps blobs out of the dotenv *.age globs; rekey + list
  recurse explicitly
- which reads back the manifest; list shows [external]; pre-commit blocks
  plaintext gradle.properties

Also fixes two latent bugs in 'secrets rekey' (never completed before, no
prior test): age-keygen refusing to overwrite key.txt, and an EXIT trap
referencing an out-of-scope local under set -u.

Tests: 80 -> 104.

Reviewed via /autoplan (CEO/Eng/DX). EGB-531.
This commit is contained in:
Brian Majewski 2026-05-26 12:42:04 -07:00
parent ac2195d830
commit 110ac514cc
7 changed files with 816 additions and 13 deletions

427
secrets
View file

@ -319,6 +319,325 @@ collect_env_files() {
[ ${#COLLECTED_FILES[@]} -gt 0 ]
}
# ─── External files (.secrets-files) — EGB-531 ─────────────────────────
#
# A committed, project-local `.secrets-files` manifest declares files that
# live OUTSIDE the project root (e.g. global Gradle properties) whose
# *designated keys* should sync. One entry per non-empty non-comment line:
#
# gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive
# <type> <path> <key> [key...]
#
# push extracts only the named keys → encrypts a subset blob under
# <project>/external/. pull decrypts it and MERGES those keys into <path>,
# preserving every unrelated line. Only type `gradle-properties` is
# supported (the type token leaves room for future types without a plugin
# framework). Parsing mirrors `.secrets-store`: no shell expansion, no
# symlink following, conservative sanitization. The merge is pure bash
# (no sed/regex) with exact-string key matching, so it is safe against
# values containing `& \ /` and against substring keys
# (beaconClerkPk vs beaconClerkPkTest).
SECRETS_FILES_NAME=".secrets-files"
SUPPORTED_EXTERNAL_TYPE="gradle-properties"
# True if the string ends in an odd number of backslashes (a Java/Gradle
# properties line-continuation). Used to skip continuation lines when
# matching keys so an unrelated continued value is never clobbered.
_trailing_bs_odd() {
local s="$1" n=0
while [ "${s%\\}" != "$s" ]; do s="${s%\\}"; n=$((n + 1)); done
[ $((n % 2)) -eq 1 ]
}
# Parse .secrets-files. Emits "<type>\t<path>\t<keys>" per valid entry on
# stdout; warnings (skipped/unsafe lines) to stderr. Conservative: rejects
# shell/control chars and `..` in the path, restricts keys to a safe
# charset. No shell expansion is applied to file content.
_parse_secrets_files_manifest() {
local file="$1"
local line lineno=0 mtype mpath mkeys
while IFS= read -r line || [ -n "$line" ]; do
lineno=$((lineno + 1))
line="${line%$'\r'}"
line="${line#"${line%%[![:space:]]*}"}"
line="${line%"${line##*[![:space:]]}"}"
[ -z "$line" ] && continue
case "$line" in '#'*) continue ;; esac
# `read -r` does not glob-expand and keeps the key list intact in $mkeys.
read -r mtype mpath mkeys <<< "$line"
if [ "$mtype" != "$SUPPORTED_EXTERNAL_TYPE" ]; then
echo "WARNING: $file line $lineno: unknown type '$mtype' (supported: $SUPPORTED_EXTERNAL_TYPE). Skipping." >&2
continue
fi
if [ -z "$mpath" ] || [ -z "$mkeys" ]; then
echo "WARNING: $file line $lineno: expected '<type> <path> <key> [key...]'. Skipping." >&2
continue
fi
# Path: allow alnum and / . _ ~ - only; reject everything else (blocks
# \$ \` ; & | ( ) * ? whitespace etc.) and reject `..` traversal.
case "$mpath" in
*[!A-Za-z0-9/._~-]* | *..*)
echo "WARNING: $file line $lineno: unsafe characters in path. Paths may contain letters, digits, / . _ - ~ only. Skipping." >&2
continue
;;
esac
# Keys: space-separated; each may contain alnum and . _ - (covers the
# systemProp. prefix). Reject anything else (=, *, control chars, ...).
case "$mkeys" in
*[!A-Za-z0-9._\ -]*)
echo "WARNING: $file line $lineno: unsafe characters in key list. Skipping." >&2
continue
;;
esac
printf '%s\t%s\t%s\n' "$mtype" "$mpath" "$mkeys"
done < "$file"
}
# Validate a writable external target path. Refuses anything that isn't a
# `gradle.properties` resolving inside $HOME, blocks `..`, and refuses
# symlinks (target file or its parent dir) — the path comes from a
# committed file, so it is attacker-controllable. Returns 0 if safe.
_validate_external_target_path() {
local p="$1"
local base; base=$(basename "$p")
if [ "$base" != "gradle.properties" ]; then
echo "ERROR: $SECRETS_FILES_NAME: target basename must be 'gradle.properties' (got '$base'). Refusing." >&2
return 1
fi
case "$p" in *..*) echo "ERROR: $SECRETS_FILES_NAME: target path may not contain '..'. Refusing." >&2; return 1 ;; esac
local home_real; home_real=$(cd -P "$HOME" 2>/dev/null && pwd -P) || home_real="$HOME"
local dir; dir=$(dirname "$p")
# Resolve the deepest existing ancestor and require it inside $HOME.
local probe="$dir"
while [ ! -e "$probe" ] && [ "$probe" != "/" ] && [ -n "$probe" ]; do
probe=$(dirname "$probe")
done
local probe_real
probe_real=$(cd -P "$probe" 2>/dev/null && pwd -P) || {
echo "ERROR: $SECRETS_FILES_NAME: cannot resolve target ancestor '$probe'. Refusing." >&2
return 1
}
case "$probe_real/" in
"$home_real"/*) ;;
*) echo "ERROR: $SECRETS_FILES_NAME: target '$p' must resolve inside \$HOME. Refusing." >&2; return 1 ;;
esac
if [ -L "$p" ]; then
echo "ERROR: $SECRETS_FILES_NAME: target '$p' is a symlink. Refusing to follow." >&2
return 1
fi
if [ -e "$dir" ] && [ -L "$dir" ]; then
echo "ERROR: $SECRETS_FILES_NAME: target dir '$dir' is a symlink. Refusing." >&2
return 1
fi
return 0
}
# Print the value of an exact key from a properties file (last definition
# wins, matching Gradle). Returns 1 if the key is absent. Pure bash; the
# key is compared by string equality (never as a regex), and the value is
# read verbatim. Skips comments and continuation lines.
_props_get() {
local file="$1" want="$2"
[ -f "$file" ] || return 1
local line t key after val="" found=0 cont=0
while IFS= read -r line || [ -n "$line" ]; do
line="${line%$'\r'}"
if [ "$cont" -eq 1 ]; then
_trailing_bs_odd "$line" && cont=1 || cont=0
continue
fi
_trailing_bs_odd "$line" && cont=1 || cont=0
t="${line#"${line%%[![:space:]]*}"}"
case "$t" in ''|'#'*|'!'*) continue ;; esac
key="${t%%[=:[:space:]]*}"
[ "$key" = "$want" ] || continue
after="${t#"$key"}"
after="${after#"${after%%[![:space:]]*}"}"
case "$after" in [=:]*) after="${after#?}" ;; esac
after="${after#"${after%%[![:space:]]*}"}"
val="$after"
found=1
done < "$file"
[ "$found" -eq 1 ] || return 1
printf '%s' "$val"
}
# Turn a manifest path token into a machine-independent blob slug.
_secrets_files_slug() {
local p="$1"
printf '%s' "${p//[!A-Za-z0-9._-]/_}"
}
# Merge managed key=value lines (from $2) into target file $1, preserving
# all unrelated lines/comments/order. Updates a managed key in place (first
# occurrence), collapses duplicates, appends new keys. Atomic + mode-safe.
merge_gradle_keys() {
local target="$1" kvfile="$2"
local -a mkey=() mval=() seen=()
local line key val i
while IFS= read -r line || [ -n "$line" ]; do
[ -n "$line" ] || continue
key="${line%%=*}"
val="${line#*=}"
mkey+=("$key"); mval+=("$val"); seen+=(0)
done < "$kvfile"
local n=${#mkey[@]}
[ "$n" -gt 0 ] || return 0
local dir; dir=$(dirname "$target")
mkdir -p "$dir"
local tmp; tmp=$(mktemp "$dir/.gradle-merge.XXXXXX") || return 1
local cont=0 t tkey matched
if [ -f "$target" ]; then
while IFS= read -r line || [ -n "$line" ]; do
local stripped="${line%$'\r'}"
if [ "$cont" -eq 1 ]; then
printf '%s\n' "$line" >> "$tmp"
_trailing_bs_odd "$stripped" && cont=1 || cont=0
continue
fi
_trailing_bs_odd "$stripped" && cont=1 || cont=0
t="${stripped#"${stripped%%[![:space:]]*}"}"
case "$t" in ''|'#'*|'!'*) printf '%s\n' "$line" >> "$tmp"; continue ;; esac
tkey="${t%%[=:[:space:]]*}"
matched=-1
for ((i = 0; i < n; i++)); do
if [ "$tkey" = "${mkey[$i]}" ]; then matched=$i; break; fi
done
if [ "$matched" -ge 0 ]; then
if [ "${seen[$matched]}" -eq 0 ]; then
printf '%s=%s\n' "${mkey[$matched]}" "${mval[$matched]}" >> "$tmp"
seen[$matched]=1
fi
else
printf '%s\n' "$line" >> "$tmp"
fi
done < "$target"
fi
for ((i = 0; i < n; i++)); do
if [ "${seen[$i]}" -eq 0 ]; then
printf '%s=%s\n' "${mkey[$i]}" "${mval[$i]}" >> "$tmp"
fi
done
# Back up the existing target before overwriting (seatbelt for the merge
# engine touching a hand-maintained file), preserve mode, atomic rename.
if [ -f "$target" ]; then
cp "$target" "$target.secrets-bak" 2>/dev/null || true
local mode
mode=$(stat -f '%Lp' "$target" 2>/dev/null || stat -c '%a' "$target" 2>/dev/null || echo 600)
chmod "$mode" "$tmp" 2>/dev/null || true
else
chmod 600 "$tmp" 2>/dev/null || true
fi
# TOCTOU recheck before the write into $HOME.
if [ -L "$target" ]; then
rm -f "$tmp"
echo "ERROR: $target became a symlink; aborting merge." >&2
return 1
fi
mv "$tmp" "$target"
}
# Encrypt the managed keys declared in <root>/.secrets-files into
# <project>/external/. Returns 0 if at least one entry was pushed, 1 if
# there is no usable manifest. Dies on unsafe targets or all-missing keys.
push_external_files() {
local root="$1" project="$2" pubkey="$3"
local manifest="$root/$SECRETS_FILES_NAME"
[ -e "$manifest" ] || return 1
if [ -L "$manifest" ]; then
echo "WARNING: $manifest is a symlink; ignoring." >&2
return 1
fi
[ -f "$manifest" ] || return 1
local pushed=0 mtype mpath mkeys
while IFS=$'\t' read -r mtype mpath mkeys; do
[ -n "$mtype" ] || continue
local expanded; expanded=$(_expand_store_path "$mpath")
if ! _validate_external_target_path "$expanded"; then
die "Refusing unsafe external target in $SECRETS_FILES_NAME: $mpath"
fi
if [ ! -f "$expanded" ]; then
echo "WARNING: source '$mpath' not found on this machine; skipping." >&2
continue
fi
local tmp; tmp=$(mktemp)
local found=0 k v
for k in $mkeys; do
if v=$(_props_get "$expanded" "$k"); then
printf '%s=%s\n' "$k" "$v" >> "$tmp"
found=$((found + 1))
else
echo "WARNING: key '$k' not found in $mpath — not synced. Set it locally first, or remove it from $SECRETS_FILES_NAME." >&2
fi
done
if [ "$found" -eq 0 ]; then
rm -f "$tmp"
die "No managed keys found in $mpath (looked for: $mkeys)."
fi
mkdir -p "$SECRETS_DIR/$project/external"
local slug; slug=$(_secrets_files_slug "$mpath")
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$slug.$mtype.age" "$tmp"
rm -f "$tmp"
info "Extracted $found key(s) from $mpath"
pushed=$((pushed + 1))
done < <(_parse_secrets_files_manifest "$manifest")
[ "$pushed" -gt 0 ]
}
# Decrypt the external blobs named by <root>/.secrets-files and merge their
# keys into the declared targets. Skips (with a warning) unsafe targets or
# missing blobs rather than aborting the whole pull.
pull_external_files() {
local root="$1" project="$2"
local manifest="$root/$SECRETS_FILES_NAME"
[ -e "$manifest" ] || return 0
if [ -L "$manifest" ]; then
echo "WARNING: $manifest is a symlink; ignoring." >&2
return 0
fi
[ -f "$manifest" ] || return 0
local mtype mpath mkeys
while IFS=$'\t' read -r mtype mpath mkeys; do
[ -n "$mtype" ] || continue
local expanded; expanded=$(_expand_store_path "$mpath")
if ! _validate_external_target_path "$expanded"; then
echo "WARNING: skipping unsafe external target: $mpath" >&2
continue
fi
local slug; slug=$(_secrets_files_slug "$mpath")
local blob="$SECRETS_DIR/$project/external/$slug.$mtype.age"
if [ ! -f "$blob" ]; then
echo "WARNING: $SECRETS_FILES_NAME names '$mpath' but no encrypted data exists in the store yet. Run 'secrets push' on a machine that has these keys. Skipping." >&2
continue
fi
local tmp; tmp=$(mktemp)
if ! age -d -i "$KEY_FILE" -o "$tmp" "$blob"; then
rm -f "$tmp"
die "Decryption failed for external target $mpath."
fi
local k
for k in $mkeys; do
grep -q "^$k=" "$tmp" 2>/dev/null || echo "WARNING: '$k' listed in $SECRETS_FILES_NAME but absent from synced data for $mpath." >&2
done
if [ ! -d "$(dirname "$expanded")" ]; then
echo "WARNING: creating $(dirname "$expanded")" >&2
fi
local count; count=$(grep -c '=' "$tmp" 2>/dev/null || echo 0)
if merge_gradle_keys "$expanded" "$tmp"; then
local klist; klist=$(printf '%s' "$mkeys" | tr ' ' ',' | sed 's/,/, /g')
info "Merged $count key(s) into $expanded ($klist)"
fi
rm -f "$tmp"
done < <(_parse_secrets_files_manifest "$manifest")
}
# Read package.json workspaces and expand globs to actual directories.
# Prints one workspace path per line (relative to the monorepo root).
get_workspaces() {
@ -353,7 +672,7 @@ install_hook() {
# Inline hook if template not found (e.g. secrets installed standalone)
cat > "$hook_dst" << 'HOOKEOF'
#!/usr/bin/env bash
BLOCKED=$(git diff --cached --name-only | grep -E '\.(env|dev\.vars)' | grep -v '\.age$' || true)
BLOCKED=$(git diff --cached --name-only | grep -E '(\.env|\.dev\.vars|gradle\.properties)' | grep -v '\.age$' || true)
if [ -n "$BLOCKED" ]; then
echo "ERROR: Plaintext secret files staged for commit:"
echo "$BLOCKED"
@ -480,8 +799,11 @@ cmd_push() {
local pubkey
pubkey=$(get_pubkey)
if ! push_dir_to_project "$PWD" "$project" "$pubkey"; then
die "No secret files (.env, .env.*, .dev.vars) found in $PWD"
local did=0
if push_dir_to_project "$PWD" "$project" "$pubkey"; then did=1; fi
if push_external_files "$PWD" "$project" "$pubkey"; then did=1; fi
if [ "$did" -eq 0 ]; then
die "No secret files (.env, .env.*, .dev.vars) or $SECRETS_FILES_NAME entries found in $PWD"
fi
commit_and_push_secrets "update $project"
@ -522,6 +844,12 @@ cmd_push_workspaces() {
fi
done <<< "$workspaces"
# External files (.secrets-files) are monorepo-root-scoped, like
# .secrets-store — handle once, not per-workspace.
if push_external_files "$root" "$monorepo_name" "$pubkey"; then
total=$((total + 1))
fi
if [ "$total" -eq 0 ]; then
die "No secret files found in any workspace"
fi
@ -569,6 +897,9 @@ cmd_pull() {
info "Decrypted $count file(s) into $target_dir"
# Merge any external files (.secrets-files) declared in this project.
pull_external_files "$PWD" "$project"
# Reinstall hook if missing
if [ ! -x "$SECRETS_DIR/.git/hooks/pre-commit" ]; then
install_hook
@ -644,6 +975,12 @@ cmd_pull_workspaces() {
fi
done <<< "$workspaces"
# External files (.secrets-files) are monorepo-root-scoped — once.
if [ -f "$root/$SECRETS_FILES_NAME" ] && [ ! -L "$root/$SECRETS_FILES_NAME" ]; then
pull_external_files "$root" "$monorepo_name"
total=$((total + 1))
fi
if [ "$total" -eq 0 ]; then
die "No secrets found for any workspace in $monorepo_name"
fi
@ -674,6 +1011,14 @@ cmd_list() {
echo " $(basename "$f" .age)"
found=1
done
# External files live in a subdir, invisible to the globs above.
if [ -d "${dir}external" ]; then
for f in "${dir}external"/*.age; do
[ -f "$f" ] || continue
echo " [external] $(basename "$f" .age)"
found=1
done
fi
done
if [ "$found" -eq 0 ]; then
@ -720,7 +1065,10 @@ cmd_rekey() {
# Create temp dir with cleanup trap
local tmpdir
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT INT TERM
# `${tmpdir:-}` guards against `set -u` if the trap somehow fires after the
# function returns (the local would be out of scope). Normal completion
# cleans up explicitly and clears the trap at the end of this function.
trap 'rm -rf "${tmpdir:-}"' EXIT INT TERM
info "Decrypting all files with current key..."
@ -741,6 +1089,20 @@ cmd_rekey() {
fi
file_count=$((file_count + 1))
done
# External files live in a subdir; rekey them too or they become
# undecryptable after rotation.
if [ -d "${dir}external" ]; then
mkdir -p "$tmpdir/$project/external"
for f in "${dir}external"/*.age; do
[ -f "$f" ] || continue
local ename
ename=$(basename "$f" .age)
if ! age -d -i "$KEY_FILE" -o "$tmpdir/$project/external/$ename" "$f"; then
die "Decryption failed for $project/external/$ename. Rekey aborted. Old key preserved."
fi
file_count=$((file_count + 1))
done
fi
done
if [ "$file_count" -eq 0 ]; then
@ -749,8 +1111,12 @@ cmd_rekey() {
info "Decrypted $file_count file(s). Generating new key pair..."
# Generate new key (overwrites old)
age-keygen -o "$KEY_FILE" 2>&1
# Generate the new key. age-keygen refuses to overwrite an existing
# file, so generate into the temp dir and move it into place only on
# success — the old key stays intact if generation fails.
local newkey="$tmpdir/key.txt.new"
age-keygen -o "$newkey" 2>&1
mv "$newkey" "$KEY_FILE"
local pubkey
pubkey=$(get_pubkey)
@ -768,6 +1134,15 @@ cmd_rekey() {
name=$(basename "$f")
age -r "$pubkey" -o "$SECRETS_DIR/$project/${name}.age" "$f"
done
if [ -d "${dir}external" ]; then
mkdir -p "$SECRETS_DIR/$project/external"
for f in "${dir}external"/*; do
[ -f "$f" ] || continue
local ename
ename=$(basename "$f")
age -r "$pubkey" -o "$SECRETS_DIR/$project/external/${ename}.age" "$f"
done
fi
done
# Commit and push
@ -787,6 +1162,11 @@ cmd_rekey() {
echo ""
echo "WARNING: Old ciphertext remains in git history."
echo "For full rotation, create a fresh repo."
# Clean up the plaintext temp dir while $tmpdir is still in scope, then
# drop the trap so it can't fire (and error under set -u) post-return.
rm -rf "$tmpdir"
trap - EXIT INT TERM
}
cmd_clear() {
@ -893,6 +1273,22 @@ cmd_which() {
resolve_store
echo "store: $SECRETS_DIR"
echo "source: $STORE_SOURCE"
# Read back any external-file manifest in cwd (validates the format and
# gives the user a way to confirm it parsed, since there's no add-file
# command). Skips symlinked manifests.
local manifest="$PWD/$SECRETS_FILES_NAME"
if [ -f "$manifest" ] && [ ! -L "$manifest" ]; then
local mtype mpath mkeys printed=0
while IFS=$'\t' read -r mtype mpath mkeys; do
[ -n "$mtype" ] || continue
if [ "$printed" -eq 0 ]; then
echo "external files ($SECRETS_FILES_NAME at $manifest):"
printed=1
fi
echo " $mtype $mpath $mkeys"
done < <(_parse_secrets_files_manifest "$manifest")
fi
}
cmd_help() {
@ -946,6 +1342,25 @@ Workspaces:
<monorepo>/<workspace-path>/ in the secrets repo. Root secret files
are stored under <monorepo>/ directly. Requires jq.
External files (.secrets-files):
Sync specific keys from files OUTSIDE the project root (e.g. global
Gradle properties). Create a committed .secrets-files in the project
root, one entry per line:
# <type> <path> <keys...>
gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive
On push, the named keys are extracted from <path> and encrypted under
<project>/external/ in the store. On pull, they are MERGED back into
<path>, leaving all unrelated keys untouched. Only type
'gradle-properties' is supported; the target basename must be
'gradle.properties' and must resolve inside $HOME. Run 'secrets which'
from the project to confirm the manifest parsed.
Note: merged keys are written as permanent plaintext into the target
file (suitable for publishable/low-secrecy values). 'secrets clear'
does NOT remove them.
Environment:
SECRETS_DIR Path to secrets repo (default: ~/.secrets). See also
the .secrets-store file and --store flag above.