diff --git a/CLAUDE.md b/CLAUDE.md index 0da5504..ede0401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,9 +16,14 @@ cd ~/my-project && ./secrets pull # Pull + decrypt .env* files ```bash brew install bats-core -bats test/secrets.bats +bats test/ # runs secrets.bats + manifest.bats ``` +**bash 3.2 assertion gotcha:** bats runs under system bash 3.2, where a +failing `[[ ]]` mid-test does NOT fail the test (the ERR trap skips `[[` +compound commands). Every standalone `[[ ... ]]` assertion MUST end with +`|| false`. Single-bracket `[ ]` assertions are unaffected. + ## Architecture Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rekey. @@ -26,6 +31,7 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek - Encryption: `age` with key files (not passphrases — age passphrases are non-scriptable) - Storage: Private git repo at `~/.secrets/` - Convention: Tracks `.env`, `.env.*`, and `.dev.vars` (not `.envrc`, `.environment-*`) +- Manifest (EGB-677 stage 1): committed `.secrets.json` is the source of truth for what syncs — `dotenv[]` (project-relative, nested ok, `@` allowed; rail rejects `..`/absolute/symlink) + `external[]` (`properties`/`file`). Push discovery auto-adds (gated by committed `options.autoAdd`, default ON; `--frozen`/`--dry-run` overrides), bootstraps the manifest on first push (written only after ≥1 blob encrypts), and absorbs a legacy `.secrets-files` (gradle-properties → `properties`; on pull the legacy file is superseded with a warning). v1 store layout unchanged in stage 1: nested entries land at `/.age`; `properties` blobs keep the legacy `.gradle-properties.age` suffix until the stage-2 store migration. jq is a hard dep only when a manifest exists/is written; manifest-less projects run jq-free (manifest features skipped with a notice). `check_cmd` prints platform-aware install hints. - External files: `.secrets-files` manifest tracks designated keys from files outside the project (e.g. `~/.gradle/gradle.properties`, merged not overwritten — EGB-531) and whole binary files (type `file`, e.g. an Android upload keystore — EGB-652); see below - Workspaces: `--workspaces` flag reads `package.json` workspaces, requires `jq` - Safety: Pre-commit hook rejects plaintext secret files (`.env`, `.dev.vars`, `gradle.properties`) @@ -34,10 +40,11 @@ Single bash script (`secrets`) with subcommands: init, push, pull, list, rm, rek ## Project Structure ``` -secrets # CLI script (~600 lines bash) +secrets # CLI script (~2000 lines bash) hooks/pre-commit # Pre-commit hook template test/ - secrets.bats # bats-core test suite (126 tests) + secrets.bats # bats-core test suite (133 tests) + manifest.bats # EGB-677 .secrets.json manifest tests (41 tests) test_helper.bash # Shared setup/teardown README.md # User-facing documentation CLAUDE.md # This file diff --git a/README.md b/README.md index d68e566..f7e754d 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,10 @@ secrets clear |---------|-------------| | `secrets init` | Create the `~/.secrets/` repo and generate an encryption key | | `secrets push` | Encrypt secret files in the current directory and upload them | +| `secrets push --frozen` | Sync only what `.secrets.json` declares (skip auto-add) | +| `secrets push --dry-run` | Show what would be added/synced without changing anything | | `secrets pull` | Download and decrypt secret files into the current directory | +| `secrets add ` | Declare a project-relative file in `.secrets.json` | | `secrets clear` | Delete plaintext secret files from the current directory | | `secrets run ` | Pull secrets, run a command, then clear secrets when it exits | | `secrets list` | Show all projects that have stored secrets | @@ -176,6 +179,32 @@ When you run `secrets push` or `secrets pull` without specifying a project name, You can also specify a name explicitly: `secrets push myapp`. +### The manifest (`.secrets.json`) + +Every project gets a committed `.secrets.json` at its root declaring exactly what syncs — the single source of truth `push` and `pull` operate from (requires `jq`): + +```json +{ + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", ".env.staging", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] +} +``` + +You rarely write it by hand: + +- **Auto-add (default):** `secrets push` discovers conventional files (`.env`, `.env.*`, `.dev.vars` — plus `package.json` workspace dirs once a manifest exists) and adds them to the manifest with an `==>` notice. Commit the manifest so other machines pick it up. +- **Explicit mode:** set `"options": {"autoAdd": false}` (a committed, team-shared setting) and `push` only syncs declared entries, warning about undeclared files. `secrets add ` is then the only manifest writer. Per-invocation: `push --frozen` (declared-only once) and `push --dry-run` (preview). +- `dotenv` paths are project-relative — nested monorepo paths like `packages/@acme/web/.env` are welcome; `..`, absolute paths, and symlinked manifests are refused. +- On the other machine, `secrets pull` restores exactly what the committed manifest declares, creating nested directories as needed. + +Projects without a manifest keep working exactly as before (and work without `jq`); the first `push` bootstraps one for you. + ### secrets run `secrets run` is a **pull → run → clear** pipeline: it runs `secrets pull` to decrypt the latest files into your project, executes your command, then runs `secrets clear` when that command finishes. Plaintext `.env` / `.dev.vars` files exist only while your command is running. @@ -352,25 +381,40 @@ Requires `jq` (`brew install jq`). Some credentials don't live in your project at all. Android builds, for example, read keys from `~/.gradle/gradle.properties` — a global file, outside any project, shared by every Gradle project on your machine (the project's own `gradle.properties` is git-tracked, so it's the wrong home for secrets). `secrets` can sync specific keys from such a file without touching the unrelated keys around them. -You declare what to sync in a committed `.secrets-files` manifest at your project root, one entry per line: +You declare what to sync in the `external` array of your committed `.secrets.json`: -``` -# -gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive +```json +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} ``` -- **type** — `gradle-properties` (sync named keys) or `file` (sync the whole file — see below). -- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `gradle-properties` the basename must be `gradle.properties`. -- **keys** — the property names to sync (`gradle-properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. +- **type** — `properties` (sync named keys from a Java-properties-style file) or `file` (sync the whole file — see below). +- **path** — absolute or `~/`-relative; must resolve inside `$HOME`. For `properties` the basename must end in `.properties`. +- **keys** — the property names to sync (`properties` only). Only these keys are read on push and merged on pull; everything else in the file is left alone. `file` entries take no keys. + +> **Legacy `.secrets-files`:** older projects declared these entries in a line-based `.secrets-files`. It still parses, and the next `secrets push` absorbs its entries into `.secrets.json` (type `gradle-properties` becomes `properties`) — after that the legacy file is superseded and can be deleted. #### Syncing to a second machine -On the machine that already has the keys set: +On the machine that already has the keys set, add the entry to `.secrets.json` (create the file if the project doesn't have one yet): ```bash cd ~/myapp -echo "gradle-properties ~/.gradle/gradle.properties beaconClerkPkTest beaconClerkPkLive" > .secrets-files -git add .secrets-files && git commit -m "sync gradle Clerk keys" +cat > .secrets.json <<'EOF' +{ + "version": 2, + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest", "beaconClerkPkLive"] } + ] +} +EOF +git add .secrets.json && git commit -m "sync gradle Clerk keys" secrets push # ==> Extracted 2 key(s) from ~/.gradle/gradle.properties ``` @@ -393,9 +437,8 @@ secrets pull Some external secrets are whole binary files — an Android upload keystore, a certificate. The `file` type syncs the file verbatim (binary-safe, encrypted with age like everything else): -``` -# -file ~/keystores/beacon-upload.keystore +```json +{ "type": "file", "path": "~/keystores/beacon-upload.keystore" } ``` On `secrets push` the file is encrypted into `/external/`. On `secrets pull` it is restored to the same path with mode `600`; if a different version already exists there, it is backed up to `.secrets-bak` first. The same path rules apply (inside `$HOME`, no `..`, no symlinks). Like merged Gradle keys, restored files are permanent plaintext on disk — `secrets clear` does not remove them. @@ -444,10 +487,14 @@ For complete rotation with no historical exposure, create a fresh `~/.secrets/` **"Fast-forward pull failed"** — Someone else pushed secrets while you had local changes. Run `secrets pull` first, then retry your push. +**".secrets.json: invalid JSON" / "manifest version N is not supported"** — The committed manifest is malformed or written by a newer `secrets`. The error names the file; fix the syntax, or update the tool (`git pull` in the tool's clone). + +**"'jq' is not installed"** — Manifest features need `jq`. The error prints the install command for your platform. Manifest-less projects work without it. + ## Development ```bash -# Run the test suite (126 tests) +# Run the test suite (174 tests across both files) brew install bats-core -bats test/secrets.bats +bats test/ ``` diff --git a/secrets b/secrets index 1e75d98..7e01e91 100755 --- a/secrets +++ b/secrets @@ -37,7 +37,18 @@ die() { echo "ERROR: $*" >&2; exit 1; } info() { echo "==> $*"; } check_cmd() { - command -v "$1" >/dev/null 2>&1 || die "'$1' is not installed. Run: brew install $1" + command -v "$1" >/dev/null 2>&1 && return + # Platform-aware install hint — hardcoding brew is wrong guidance on a + # Linux box or CI runner (EGB-677 DX review). + local hint="install '$1' with your package manager" + if command -v brew >/dev/null 2>&1; then + hint="brew install $1" + elif command -v apt-get >/dev/null 2>&1; then + hint="sudo apt-get install $1" + elif command -v dnf >/dev/null 2>&1; then + hint="sudo dnf install $1" + fi + die "'$1' is not installed. Run: $hint" } check_initialized() { @@ -1216,6 +1227,12 @@ cmd_push() { pubkey=$(get_pubkey) # ── Manifest read (validated; absence = bootstrap) ── + # jq is required only when a manifest exists (authoritative, can't be + # ignored) or is being written. Without jq on a manifest-less project, + # manifest features are skipped with a notice — clone-and-run for v1 + # users survives. + local have_jq=true + command -v jq >/dev/null 2>&1 || have_jq=false local manifest="$PWD/$SECRETS_JSON_NAME" local have_manifest=false auto_add=true declared="" if [ -e "$manifest" ]; then @@ -1312,8 +1329,11 @@ cmd_push() { # Two independent reasons to write: dotenv auto-adds, and absorbing a # legacy .secrets-files (gradle-properties → properties) so the two # external sources converge on the manifest. + if [ "$have_jq" = false ]; then + echo "NOTE: jq not found — skipping $SECRETS_JSON_NAME manifest features (auto-add, absorb). Install jq to enable them." >&2 + fi local absorbed_json="[]" n_absorbed=0 - if [ "$frozen" = false ]; then + if [ "$frozen" = false ] && [ "$have_jq" = true ]; then absorbed_json=$(_legacy_absorb_json "$PWD") n_absorbed=$(printf '%s' "$absorbed_json" | jq 'length') fi @@ -1321,7 +1341,7 @@ cmd_push() { if [ -n "$to_add" ] && { [ "$auto_add" = true ] || [ "$have_manifest" = false ]; }; then write_adds=true fi - if [ "$did" -eq 1 ] && [ "$frozen" = false ] \ + if [ "$did" -eq 1 ] && [ "$frozen" = false ] && [ "$have_jq" = true ] \ && { [ "$write_adds" = true ] || [ "$n_absorbed" -gt 0 ]; }; then local add_json="[]" [ "$write_adds" = true ] && add_json=$(printf '%s' "$to_add" | jq -R -s 'split("\n") | map(select(length > 0))') @@ -1905,21 +1925,48 @@ secrets — encrypted secret file sync between machines Usage: secrets init Initialize the secrets repo and generate an age key secrets push [project] Encrypt secret files and push to the secrets repo + secrets push --frozen Sync only manifest-declared files (skip auto-add) + secrets push --dry-run Show what would be added/synced; change nothing secrets push -w|--workspaces Push secrets from all workspaces in package.json secrets pull [project] Pull and decrypt secret files into current directory secrets pull -w|--workspaces Pull secrets into all workspaces from package.json + secrets add Declare a project-relative file in .secrets.json secrets clear Remove plaintext secret files from current directory secrets clear -w|--workspaces Clear secrets from all workspaces in package.json secrets run [-w] Pull secrets, run command, clear secrets on exit secrets list List all projects and their secret files secrets rm Remove a project's secrets from the repo secrets rekey Re-encrypt all secrets with a new key - secrets which Show the active store path and which rule chose it + secrets which Show the active store, manifest, and external entries secrets where Alias for `which` secrets status Alias for `which` Tracked files: .env, .env.*, .dev.vars +Manifest (.secrets.json): + A committed project-root manifest declaring everything the project + syncs (requires jq). `secrets push` discovers conventional files and + auto-adds them with a notice; set {"options":{"autoAdd":false}} to + require explicit `secrets add` instead. Dotenv paths are project- + relative (nested workspace paths welcome); external entries use + {"type":"properties"|"file","path":...,"keys":[...]}: + + { + "version": 2, + "options": { "autoAdd": true }, + "dotenv": [".env", "packages/web/.env.development"], + "external": [ + { "type": "properties", "path": "~/.gradle/gradle.properties", + "keys": ["beaconClerkPkTest"] }, + { "type": "file", "path": "~/keystores/upload.keystore" } + ] + } + + A legacy .secrets-files is absorbed into .secrets.json on the next + push (gradle-properties entries become type "properties") and can be + deleted afterwards. Without jq, manifest-less projects keep working; + manifest features are skipped with a notice. + If [project] is omitted, it is derived from the current directory's git remote (if available) or the directory name. diff --git a/test/manifest.bats b/test/manifest.bats index f23a5aa..822c1ca 100644 --- a/test/manifest.bats +++ b/test/manifest.bats @@ -462,3 +462,48 @@ m_file_src() { mkdir -p "$HOME/keystores"; printf 'KS\x00\x01\x02\xffDATA\n' > [ -f packages/api/.dev.vars ] [ "$(cat packages/api/.dev.vars)" = "API=1" ] } + +# ─── E: jq gating + install hints + help ─────────────────────────────── + +# Helper: PATH with age but without jq. macOS ships /usr/bin/jq, so +# /usr/bin must be excluded too — needed tools are symlinked explicitly. +m_nojq_path() { + local fake="$TEST_TMPDIR/nojq-bin" + mkdir -p "$fake" + local t + for t in age age-keygen git basename dirname mktemp grep sed tr cut cksum stat head tail sort uniq wc env touch find diff cmp; do + command -v "$t" >/dev/null 2>&1 && ln -sf "$(command -v "$t")" "$fake/$t" + done + rm -f "$fake/jq" + echo "$fake:/bin" +} + +@test "manifest-less push works without jq (manifest features skipped)" { + init_with_remote + create_project_dir nojqproj + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 0 ] + [ -f "$SECRETS_DIR/nojqproj/.env.age" ] + [ ! -f ".secrets.json" ] + [[ "$output" == *"jq"* ]] || false +} + +@test "push dies with an install hint when a manifest exists but jq is missing" { + init_with_remote + create_project_dir needjq + printf '{"version":2,"dotenv":[".env"]}\n' > .secrets.json + local p; p=$(m_nojq_path) + run env PATH="$p" "$SECRETS_BIN" push + [ "$status" -eq 1 ] + [[ "$output" == *"'jq' is not installed"* ]] || false +} + +@test "help documents add, --frozen, --dry-run and the manifest" { + run "$SECRETS_BIN" help + [ "$status" -eq 0 ] + [[ "$output" == *"secrets add"* ]] || false + [[ "$output" == *"--frozen"* ]] || false + [[ "$output" == *"--dry-run"* ]] || false + [[ "$output" == *".secrets.json"* ]] || false +}