From e0474f3ce9041801226c8090006016075f6ce966 Mon Sep 17 00:00:00 2001 From: Brian Majewski Date: Sun, 7 Jun 2026 06:43:45 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20'file'=20external=20type=20=E2=80=94=20?= =?UTF-8?q?whole-file=20sync=20for=20.secrets-files=20(EGB-652)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Built for the Beacon Android upload keystore: binary files outside the project root can now ride the encrypted store. - manifest: 'file ' (no keys; keys present = rejected loudly) - push: encrypts the file verbatim (age is binary-safe) - pull: restores next to target (atomic same-fs mv), mode 600, TOCTOU symlink recheck, divergent existing target backed up to .secrets-bak - _validate_external_target_path parameterized by type (basename restriction stays gradle-properties-only; $HOME/../symlink rails apply to both) - README/help/CLAUDE docs + 7 bats tests (133/133 pass) --- CHANGELOG.md | 12 +++++ README.md | 19 +++++-- VERSION | 2 +- secrets | 132 ++++++++++++++++++++++++++++++++++------------ test/secrets.bats | 101 +++++++++++++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46ab88e..d8e6e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to a four-digit MAJOR.MINOR.PATCH.MICRO version scheme. +## [0.3.0.0] - 2026-06-07 + +### Added + +- **`file` external type (EGB-652)** — `.secrets-files` can now sync whole + files outside the project root (binary-safe; built for the Beacon Android + upload keystore): `file ~/keystores/beacon-upload.keystore`. Push encrypts + the file verbatim into `/external/`; pull restores it with mode + 600, backing up a divergent existing target to `.secrets-bak`. Same + path safety rails as `gradle-properties` (inside `$HOME`, no `..`, no + symlinks) minus the basename restriction. 7 new bats tests. + ## [0.2.1.0] - 2026-06-05 ### Fixed diff --git a/README.md b/README.md index d594f57..d68e566 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ flowchart TD Files like `.envrc` (direnv) and `.environment-*` are intentionally **not** tracked. -Beyond project files, `secrets` can also sync designated keys from files that live *outside* the project — like `~/.gradle/gradle.properties` — merging them in without clobbering unrelated keys. See [External files (Gradle properties)](#external-files-gradle-properties). +Beyond project files, `secrets` can also sync files that live *outside* the project — designated keys from `~/.gradle/gradle.properties` (merged without clobbering unrelated keys), or whole files like an Android upload keystore. See [External files (Gradle properties)](#external-files-gradle-properties). ## Prerequisites @@ -359,9 +359,9 @@ You declare what to sync in a committed `.secrets-files` manifest at your projec gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive ``` -- **type** — `gradle-properties` (the only supported type today). -- **path** — absolute or `~/`-relative. The basename must be `gradle.properties` and must resolve inside `$HOME`. -- **keys** — the property names to sync. Only these keys are read on push and merged on pull; everything else in the file is left alone. +- **type** — `gradle-properties` (sync named keys) or `file` (sync the whole file — see below). +- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `gradle-properties` the basename must be `gradle.properties`. +- **keys** — the property names to sync (`gradle-properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. #### Syncing to a second machine @@ -389,6 +389,17 @@ secrets pull > **Note:** unlike `.env` files, merged Gradle keys are written as **permanent plaintext** into the target file — `secrets clear` does **not** remove them. This is appropriate for publishable / low-secrecy values (like Clerk publishable keys, `pk_*`). For high-value secrets that should never sit on disk, use `secrets run` with a `.env` instead. +#### Whole files (`file` type) + +Some external secrets are whole binary files — an Android upload keystore, a certificate. The `file` type syncs the file verbatim (binary-safe, encrypted with age like everything else): + +``` +# +file ~/keystores/beacon-upload.keystore +``` + +On `secrets push` the file is encrypted into `/external/`. On `secrets pull` it is restored to the same path with mode `600`; if a different version already exists there, it is backed up to `.secrets-bak` first. The same path rules apply (inside `$HOME`, no `..`, no symlinks). Like merged Gradle keys, restored files are permanent plaintext on disk — `secrets clear` does not remove them. + ## Safety features - **`secrets run` auto-clears** — plaintext files are deleted when the command exits, errors, or is interrupted with Ctrl-C diff --git a/VERSION b/VERSION index 123b55d..1da00ae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.1.0 +0.3.0.0 diff --git a/secrets b/secrets index 97d9640..508e5de 100755 --- a/secrets +++ b/secrets @@ -330,16 +330,24 @@ collect_env_files() { # # push extracts only the named keys → encrypts a subset blob under # /external/. pull decrypts it and MERGES those keys into , -# preserving every unrelated line. Only type `gradle-properties` is -# supported (the type token leaves room for future types without a plugin -# framework). Parsing mirrors `.secrets-store`: no shell expansion, no -# symlink following, conservative sanitization. The merge is pure bash -# (no sed/regex) with exact-string key matching, so it is safe against -# values containing `& \ /` and against substring keys +# preserving every unrelated line. Parsing mirrors `.secrets-store`: no +# shell expansion, no symlink following, conservative sanitization. The +# merge is pure bash (no sed/regex) with exact-string key matching, so it +# is safe against values containing `& \ /` and against substring keys # (beaconClerkPk vs beaconClerkPkTest). +# +# A second type, `file` (EGB-652), syncs a WHOLE file outside the project +# root (e.g. an Android upload keystore) — binary-safe via age, no keys: +# +# file ~/keystores/beacon-upload.keystore +# +# push encrypts the file verbatim; pull restores it (mode 600, existing +# target backed up to .secrets-bak first). Same path rules as +# gradle-properties (inside $HOME, no `..`, no symlinks) minus the +# basename restriction. SECRETS_FILES_NAME=".secrets-files" -SUPPORTED_EXTERNAL_TYPE="gradle-properties" +SUPPORTED_EXTERNAL_TYPES="gradle-properties file" # True if the string ends in an odd number of backslashes (a Java/Gradle # properties line-continuation). Used to skip continuation lines when @@ -366,14 +374,28 @@ _parse_secrets_files_manifest() { case "$line" in '#'*) continue ;; esac # `read -r` does not glob-expand and keeps the key list intact in $mkeys. read -r mtype mpath mkeys <<< "$line" - if [ "$mtype" != "$SUPPORTED_EXTERNAL_TYPE" ]; then - echo "WARNING: $file line $lineno: unknown type '$mtype' (supported: $SUPPORTED_EXTERNAL_TYPE). Skipping." >&2 - continue - fi - if [ -z "$mpath" ] || [ -z "$mkeys" ]; then - echo "WARNING: $file line $lineno: expected ' [key...]'. Skipping." >&2 - continue - fi + case "$mtype" in + gradle-properties) + if [ -z "$mpath" ] || [ -z "$mkeys" ]; then + echo "WARNING: $file line $lineno: expected 'gradle-properties [key...]'. Skipping." >&2 + continue + fi + ;; + file) + if [ -z "$mpath" ]; then + echo "WARNING: $file line $lineno: expected 'file '. Skipping." >&2 + continue + fi + if [ -n "$mkeys" ]; then + echo "WARNING: $file line $lineno: 'file' entries take no keys (got '$mkeys'). Skipping." >&2 + continue + fi + ;; + *) + echo "WARNING: $file line $lineno: unknown type '$mtype' (supported: $SUPPORTED_EXTERNAL_TYPES). Skipping." >&2 + continue + ;; + esac # Path: allow alnum and / . _ ~ - only; reject everything else (blocks # \$ \` ; & | ( ) * ? whitespace etc.) and reject `..` traversal. case "$mpath" in @@ -394,14 +416,16 @@ _parse_secrets_files_manifest() { done < "$file" } -# Validate a writable external target path. Refuses anything that isn't a -# `gradle.properties` resolving inside $HOME, blocks `..`, and refuses -# symlinks (target file or its parent dir) — the path comes from a -# committed file, so it is attacker-controllable. Returns 0 if safe. +# Validate a writable external target path. Requires the path to resolve +# inside $HOME, blocks `..`, and refuses symlinks (target file or its +# parent dir) — the path comes from a committed file, so it is +# attacker-controllable. For type `gradle-properties` the basename must be +# 'gradle.properties' (type `file` allows any basename — the whole file is +# replaced, never merged). Returns 0 if safe. _validate_external_target_path() { - local p="$1" + local p="$1" mtype="${2:-gradle-properties}" local base; base=$(basename "$p") - if [ "$base" != "gradle.properties" ]; then + if [ "$mtype" = "gradle-properties" ] && [ "$base" != "gradle.properties" ]; then echo "ERROR: $SECRETS_FILES_NAME: target basename must be 'gradle.properties' (got '$base'). Refusing." >&2 return 1 fi @@ -563,13 +587,22 @@ push_external_files() { while IFS=$'\t' read -r mtype mpath mkeys; do [ -n "$mtype" ] || continue local expanded; expanded=$(_expand_store_path "$mpath") - if ! _validate_external_target_path "$expanded"; then + if ! _validate_external_target_path "$expanded" "$mtype"; then die "Refusing unsafe external target in $SECRETS_FILES_NAME: $mpath" fi if [ ! -f "$expanded" ]; then echo "WARNING: source '$mpath' not found on this machine; skipping." >&2 continue fi + if [ "$mtype" = "file" ]; then + # EGB-652: whole-file sync — encrypt the file verbatim (binary-safe). + mkdir -p "$SECRETS_DIR/$project/external" + local fslug; fslug=$(_secrets_files_slug "$mpath") + age -r "$pubkey" -o "$SECRETS_DIR/$project/external/$fslug.file.age" "$expanded" + info "Encrypted file $mpath" + pushed=$((pushed + 1)) + continue + fi local tmp; tmp=$(mktemp) local found=0 k v for k in $mkeys; do @@ -620,7 +653,7 @@ pull_external_files() { while IFS=$'\t' read -r mtype mpath mkeys; do [ -n "$mtype" ] || continue local expanded; expanded=$(_expand_store_path "$mpath") - if ! _validate_external_target_path "$expanded"; then + if ! _validate_external_target_path "$expanded" "$mtype"; then echo "WARNING: skipping unsafe external target: $mpath" >&2 continue fi @@ -630,6 +663,30 @@ pull_external_files() { echo "WARNING: $SECRETS_FILES_NAME names '$mpath' but no encrypted data exists in the store yet. Run 'secrets push' on a machine that has these keys. Skipping." >&2 continue fi + if [ "$mtype" = "file" ]; then + # EGB-652: whole-file restore — decrypt next to the target (same + # filesystem -> atomic mv), back up any existing target, mode 600, + # TOCTOU symlink recheck before the write into $HOME. + local fdir; fdir=$(dirname "$expanded") + mkdir -p "$fdir" + local ftmp; ftmp=$(mktemp "$fdir/.secrets-file.XXXXXX") || { echo "WARNING: mktemp failed for $mpath; skipping." >&2; continue; } + if ! age -d -i "$KEY_FILE" -o "$ftmp" "$blob"; then + rm -f "$ftmp" + die "Decryption failed for external target $mpath." + fi + chmod 600 "$ftmp" 2>/dev/null || true + if [ -L "$expanded" ]; then + rm -f "$ftmp" + echo "WARNING: $expanded became a symlink; skipping restore." >&2 + continue + fi + if [ -f "$expanded" ] && ! cmp -s "$expanded" "$ftmp"; then + cp "$expanded" "$expanded.secrets-bak" 2>/dev/null || true + fi + mv "$ftmp" "$expanded" + info "Restored file $expanded" + continue + fi local tmp; tmp=$(mktemp) if ! age -d -i "$KEY_FILE" -o "$tmp" "$blob"; then rm -f "$tmp" @@ -1409,23 +1466,28 @@ Workspaces: are stored under / directly. Requires jq. External files (.secrets-files): - Sync specific keys from files OUTSIDE the project root (e.g. global - Gradle properties). Create a committed .secrets-files in the project - root, one entry per line: + Sync files (or specific keys from files) OUTSIDE the project root. + Create a committed .secrets-files in the project root, one entry per + line: # gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive + file ~/keystores/beacon-upload.keystore - On push, the named keys are extracted from and encrypted under - /external/ in the store. On pull, they are MERGED back into - , leaving all unrelated keys untouched. Only type - 'gradle-properties' is supported; the target basename must be - 'gradle.properties' and must resolve inside $HOME. Run 'secrets which' - from the project to confirm the manifest parsed. + gradle-properties: on push, the named keys are extracted from + and encrypted under /external/ in the store. On pull, they + are MERGED back into , leaving all unrelated keys untouched. + The target basename must be 'gradle.properties'. - Note: merged keys are written as permanent plaintext into the target - file (suitable for publishable/low-secrecy values). 'secrets clear' - does NOT remove them. + file: the whole file is encrypted verbatim (binary-safe — keystores, + certificates). On pull it is restored with mode 600; an existing + divergent target is backed up to .secrets-bak first. No keys. + + All targets must resolve inside $HOME (no '..', no symlinks). Run + 'secrets which' from the project to confirm the manifest parsed. + + Note: pulled external targets are permanent plaintext on disk — + 'secrets clear' does NOT remove them. Environment: SECRETS_DIR Path to secrets repo (default: ~/.secrets). See also diff --git a/test/secrets.bats b/test/secrets.bats index f5565aa..f396171 100644 --- a/test/secrets.bats +++ b/test/secrets.bats @@ -1657,3 +1657,104 @@ gradle_project() { [ "$status" -eq 1 ] [[ "$output" == *"git clone git@example.com:me/secrets-work.git"* ]] } + +# ─── EGB-652: `file` external type (whole-file sync, e.g. Android keystore) ── + +# Helper: write a small binary source file under the sandboxed HOME. +file_src() { + mkdir -p "$HOME/keystores" + printf 'KS\x00\x01\x02\xffDATA-%s\n' "${1:-v1}" > "$HOME/keystores/upload.keystore" +} + +# Helper: bind a project dir to a file entry via .secrets-files, cd into it. +file_project() { + local dir="$WORK_DIR/$1" + mkdir -p "$dir" + printf 'file ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" || exit 1 +} + +@test "EGB-652: which shows parsed file-type entry" { + init_with_remote + file_src + file_project fproj + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"file"* ]] + [[ "$output" == *"~/keystores/upload.keystore"* ]] +} + +@test "EGB-652: push encrypts a file-type entry into external/ blob" { + init_with_remote + file_src + file_project fproj + run "$SECRETS_BIN" push fproj + [ "$status" -eq 0 ] + [[ "$output" == *"Encrypted file"* ]] + run bash -c "ls $SECRETS_DIR/fproj/external/*.file.age" + [ "$status" -eq 0 ] +} + +@test "EGB-652: pull restores the file byte-identical with mode 600" { + init_with_remote + file_src v1 + file_project fproj + "$SECRETS_BIN" push fproj >/dev/null 2>&1 + cp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/reference" + rm -rf "$HOME/keystores" + run "$SECRETS_BIN" pull fproj + [ "$status" -eq 0 ] + [[ "$output" == *"Restored file"* ]] + cmp "$HOME/keystores/upload.keystore" "$TEST_TMPDIR/reference" + mode=$(stat -f '%Lp' "$HOME/keystores/upload.keystore" 2>/dev/null || stat -c '%a' "$HOME/keystores/upload.keystore") + [ "$mode" = "600" ] +} + +@test "EGB-652: pull backs up an existing divergent target before overwriting" { + init_with_remote + file_src v1 + file_project fproj + "$SECRETS_BIN" push fproj >/dev/null 2>&1 + file_src v2-local-edit + run "$SECRETS_BIN" pull fproj + [ "$status" -eq 0 ] + grep -q 'DATA-v1' "$HOME/keystores/upload.keystore" + grep -q 'DATA-v2-local-edit' "$HOME/keystores/upload.keystore.secrets-bak" +} + +@test "EGB-652: file entry with trailing keys is rejected" { + init_with_remote + file_src + local dir="$WORK_DIR/fbad"; mkdir -p "$dir" + printf 'file ~/keystores/upload.keystore strayKey\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" which + [ "$status" -eq 0 ] + [[ "$output" == *"take no keys"* ]] + # The rejected entry must not be listed as parsed (header only prints + # when at least one entry parses). + [[ "$output" != *"external files ("* ]] +} + +@test "EGB-652: file target outside HOME is refused on push" { + init_with_remote + local dir="$WORK_DIR/fout"; mkdir -p "$dir" + printf 'file /etc/hosts\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push fout + [ "$status" -ne 0 ] + [[ "$output" == *"inside \$HOME"* ]] || [[ "$output" == *"Refusing"* ]] +} + +@test "EGB-652: gradle-properties entries still work alongside a file entry" { + init_with_remote + gradle_src $'beaconClerkPkTest=pk_test_abc\n' + file_src + local dir="$WORK_DIR/fmix"; mkdir -p "$dir" + printf 'gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest\nfile ~/keystores/upload.keystore\n' > "$dir/.secrets-files" + cd "$dir" + run "$SECRETS_BIN" push fmix + [ "$status" -eq 0 ] + [[ "$output" == *"Extracted 1 key"* ]] + [[ "$output" == *"Encrypted file"* ]] +}