feat: .secrets.json manifest core — add command, rails, canonical form (EGB-677 stage 1)
- secrets add <path>: explicit manifest writer, idempotent, atomic write - _validate_dotenv_rel_path: project-relative confinement rail (no .. / absolute / shell metas; @ allowed for npm-scoped workspace dirs) - _check_manifest_file: refuses symlinks, malformed JSON (jq error with file named), unsupported schema versions (directed upgrade error) - canonical serialization: jq --sort-keys + sorted/deduped dotenv — add order produces byte-identical manifests - which: validates + summarizes the manifest (doubles as linter) - jq required only when a manifest exists/is written
This commit is contained in:
Brian Majewski
parent
6dbc4e0d01
commit
18018dbd3b
2 changed files with 272 additions and 0 deletions
147
secrets
147
secrets
|
|
@ -712,6 +712,129 @@ pull_external_files() {
|
||||||
done < <(_parse_secrets_files_manifest "$manifest")
|
done < <(_parse_secrets_files_manifest "$manifest")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ─── Manifest (.secrets.json) — EGB-677 store format v2, stage 1 ──────
|
||||||
|
#
|
||||||
|
# A committed, project-root JSON manifest declaring everything the project
|
||||||
|
# syncs. Stage 1 is ADDITIVE: the manifest is read alongside the existing
|
||||||
|
# dotenv globs and `.secrets-files`; the v1 store layout is unchanged.
|
||||||
|
#
|
||||||
|
# {
|
||||||
|
# "version": 2,
|
||||||
|
# "options": { "autoAdd": true },
|
||||||
|
# "dotenv": [".env", "packages/web/.env.development"],
|
||||||
|
# "external": [
|
||||||
|
# { "type": "properties", "path": "~/.gradle/gradle.properties",
|
||||||
|
# "keys": ["beaconClerkPk"] },
|
||||||
|
# { "type": "file", "path": "~/keystores/beacon-upload.keystore" }
|
||||||
|
# ]
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# Security posture mirrors `.secrets-store`/`.secrets-files`: symlinked
|
||||||
|
# manifests are refused, dotenv paths are confined to the project root
|
||||||
|
# (no `..`, no absolute paths, conservative charset — `@` allowed for
|
||||||
|
# npm-scoped workspace dirs), and every jq-extracted string is
|
||||||
|
# re-validated before any filesystem use. jq is required only when a
|
||||||
|
# manifest exists or is being written — manifest-less projects keep
|
||||||
|
# working with zero new dependencies.
|
||||||
|
|
||||||
|
SECRETS_JSON_NAME=".secrets.json"
|
||||||
|
MANIFEST_VERSION=2
|
||||||
|
|
||||||
|
# Validate a project-relative dotenv path from the manifest (or `secrets
|
||||||
|
# add`). The path is attacker-influenced in team repos (committed file),
|
||||||
|
# so this is a security rail, not just hygiene. Prints an error and
|
||||||
|
# returns 1 when unsafe.
|
||||||
|
_validate_dotenv_rel_path() {
|
||||||
|
local p="$1"
|
||||||
|
if [ -z "$p" ]; then
|
||||||
|
echo "ERROR: $SECRETS_JSON_NAME: empty dotenv path." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
case "$p" in
|
||||||
|
/*)
|
||||||
|
echo "ERROR: $SECRETS_JSON_NAME: '$p' is absolute — dotenv paths must be project-relative." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
*..*)
|
||||||
|
echo "ERROR: $SECRETS_JSON_NAME: '$p' contains '..' — dotenv paths must be project-relative (no traversal)." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
*[!A-Za-z0-9@/._-]*)
|
||||||
|
echo "ERROR: $SECRETS_JSON_NAME: unsafe characters in '$p'. Paths may contain letters, digits, @ / . _ - only." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Validate the manifest file itself (existence assumed checked by caller):
|
||||||
|
# refuse symlinks, malformed JSON, and unsupported schema versions — each
|
||||||
|
# with a directed error. $1 = manifest path.
|
||||||
|
_check_manifest_file() {
|
||||||
|
local manifest="$1"
|
||||||
|
if [ -L "$manifest" ]; then
|
||||||
|
die "$manifest is a symlink. Refusing to read it.
|
||||||
|
A committed symlink could point the manifest at attacker-chosen content.
|
||||||
|
Replace it with a regular file."
|
||||||
|
fi
|
||||||
|
check_cmd jq
|
||||||
|
local jq_err
|
||||||
|
if ! jq_err=$(jq -e . "$manifest" 2>&1 >/dev/null); then
|
||||||
|
die "$manifest: invalid JSON.
|
||||||
|
$jq_err
|
||||||
|
Fix the syntax (or delete the file and re-run 'secrets add' / 'secrets push')."
|
||||||
|
fi
|
||||||
|
local ver
|
||||||
|
ver=$(jq -r '.version // "missing"' "$manifest")
|
||||||
|
if [ "$ver" != "$MANIFEST_VERSION" ]; then
|
||||||
|
die "$manifest: manifest version $ver is not supported.
|
||||||
|
This client supports version $MANIFEST_VERSION.
|
||||||
|
If the manifest was written by a newer secrets, upgrade this machine:
|
||||||
|
git -C $SCRIPT_DIR pull"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Canonically (re)write the manifest: sorted keys, sorted+deduped dotenv,
|
||||||
|
# atomic tmp+mv in the project dir. stdin = the new JSON document.
|
||||||
|
_write_manifest_canonical() {
|
||||||
|
local manifest="$1"
|
||||||
|
local dir; dir=$(dirname "$manifest")
|
||||||
|
local tmp; tmp=$(mktemp "$dir/.secrets-json.XXXXXX") || return 1
|
||||||
|
if ! jq --sort-keys '.dotenv |= ((. // []) | unique | sort)' > "$tmp"; then
|
||||||
|
rm -f "$tmp"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
mv "$tmp" "$manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
# `secrets add <path>` — the explicit manifest writer. Creates
|
||||||
|
# .secrets.json on first use; validates and dedupes thereafter.
|
||||||
|
cmd_add() {
|
||||||
|
local p="${1:-}"
|
||||||
|
[ -n "$p" ] || die "Usage: secrets add <project-relative-path>"
|
||||||
|
# Normalize a leading ./
|
||||||
|
p="${p#./}"
|
||||||
|
_validate_dotenv_rel_path "$p" || exit 1
|
||||||
|
if [ ! -f "$PWD/$p" ]; then
|
||||||
|
die "'$p' not found in $PWD.
|
||||||
|
Create the file first, then re-run: secrets add $p"
|
||||||
|
fi
|
||||||
|
check_cmd jq
|
||||||
|
local manifest="$PWD/$SECRETS_JSON_NAME"
|
||||||
|
if [ -e "$manifest" ]; then
|
||||||
|
_check_manifest_file "$manifest"
|
||||||
|
jq --arg p "$p" '.dotenv = ((.dotenv // []) + [$p])' "$manifest" \
|
||||||
|
| _write_manifest_canonical "$manifest" || die "Failed to update $manifest"
|
||||||
|
else
|
||||||
|
jq -n --arg p "$p" '{version: '"$MANIFEST_VERSION"', dotenv: [$p]}' \
|
||||||
|
| _write_manifest_canonical "$manifest" || die "Failed to write $manifest"
|
||||||
|
fi
|
||||||
|
info "Added '$p' to $SECRETS_JSON_NAME"
|
||||||
|
info "Commit the manifest so other machines pick it up. To undo: edit $SECRETS_JSON_NAME and remove the entry."
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── End manifest ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
# Read package.json workspaces and expand globs to actual directories.
|
# Read package.json workspaces and expand globs to actual directories.
|
||||||
# Prints one workspace path per line (relative to the monorepo root).
|
# Prints one workspace path per line (relative to the monorepo root).
|
||||||
get_workspaces() {
|
get_workspaces() {
|
||||||
|
|
@ -1397,6 +1520,29 @@ cmd_which() {
|
||||||
echo "store: $SECRETS_DIR"
|
echo "store: $SECRETS_DIR"
|
||||||
echo "source: $STORE_SOURCE"
|
echo "source: $STORE_SOURCE"
|
||||||
|
|
||||||
|
# v2 manifest (.secrets.json): validate and summarize. Validation here
|
||||||
|
# is deliberately fatal (symlink / malformed / unsupported version) so
|
||||||
|
# `secrets which` doubles as the manifest linter.
|
||||||
|
local json_manifest="$PWD/$SECRETS_JSON_NAME"
|
||||||
|
if [ -e "$json_manifest" ]; then
|
||||||
|
_check_manifest_file "$json_manifest"
|
||||||
|
echo "manifest ($SECRETS_JSON_NAME at $json_manifest):"
|
||||||
|
local entry
|
||||||
|
while IFS= read -r entry; do
|
||||||
|
[ -n "$entry" ] || continue
|
||||||
|
if _validate_dotenv_rel_path "$entry"; then
|
||||||
|
echo " dotenv $entry"
|
||||||
|
else
|
||||||
|
echo " dotenv $entry [UNSAFE — will be refused]"
|
||||||
|
fi
|
||||||
|
done < <(jq -r '.dotenv // [] | .[]' "$json_manifest")
|
||||||
|
local etype epath ekeys
|
||||||
|
while IFS=$'\t' read -r etype epath ekeys; do
|
||||||
|
[ -n "$etype" ] || continue
|
||||||
|
echo " $etype $epath $ekeys"
|
||||||
|
done < <(jq -r '.external // [] | .[] | [.type, .path, ((.keys // []) | join(" "))] | @tsv' "$json_manifest")
|
||||||
|
fi
|
||||||
|
|
||||||
# Read back any external-file manifest in cwd (validates the format and
|
# Read back any external-file manifest in cwd (validates the format and
|
||||||
# gives the user a way to confirm it parsed, since there's no add-file
|
# gives the user a way to confirm it parsed, since there's no add-file
|
||||||
# command). Skips symlinked manifests.
|
# command). Skips symlinked manifests.
|
||||||
|
|
@ -1569,6 +1715,7 @@ case "${1:-help}" in
|
||||||
shift
|
shift
|
||||||
cmd_run "$@"
|
cmd_run "$@"
|
||||||
;;
|
;;
|
||||||
|
add) cmd_add "${2:-}" ;;
|
||||||
list) cmd_list ;;
|
list) cmd_list ;;
|
||||||
rm) cmd_rm "${2:-}" ;;
|
rm) cmd_rm "${2:-}" ;;
|
||||||
rekey) cmd_rekey ;;
|
rekey) cmd_rekey ;;
|
||||||
|
|
|
||||||
125
test/manifest.bats
Normal file
125
test/manifest.bats
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# EGB-677 stage 1: .secrets.json manifest — parse, rails, add, generators.
|
||||||
|
|
||||||
|
load test_helper
|
||||||
|
|
||||||
|
# ─── A: manifest core — secrets add + rails + canonical form ──────────
|
||||||
|
|
||||||
|
@test "add creates .secrets.json with version 2 and the dotenv entry" {
|
||||||
|
create_project_dir addproj
|
||||||
|
run "$SECRETS_BIN" add .env
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ -f ".secrets.json" ]
|
||||||
|
run jq -r '.version' .secrets.json
|
||||||
|
[ "$output" = "2" ]
|
||||||
|
run jq -r '.dotenv[0]' .secrets.json
|
||||||
|
[ "$output" = ".env" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add is idempotent — no duplicate entries" {
|
||||||
|
create_project_dir addproj
|
||||||
|
"$SECRETS_BIN" add .env >/dev/null
|
||||||
|
run "$SECRETS_BIN" add .env
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
run jq -r '.dotenv | length' .secrets.json
|
||||||
|
[ "$output" = "1" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add accepts nested workspace paths" {
|
||||||
|
create_project_dir addproj
|
||||||
|
mkdir -p packages/web
|
||||||
|
echo "K=v" > packages/web/.env.development
|
||||||
|
run "$SECRETS_BIN" add packages/web/.env.development
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
run jq -r '.dotenv | index("packages/web/.env.development") != null' .secrets.json
|
||||||
|
[ "$output" = "true" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add accepts npm-scoped workspace paths (@)" {
|
||||||
|
create_project_dir addproj
|
||||||
|
mkdir -p "packages/@acme/web"
|
||||||
|
echo "K=v" > "packages/@acme/web/.env"
|
||||||
|
run "$SECRETS_BIN" add "packages/@acme/web/.env"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
run jq -r '.dotenv | index("packages/@acme/web/.env") != null' .secrets.json
|
||||||
|
[ "$output" = "true" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add rejects path traversal (..)" {
|
||||||
|
create_project_dir addproj
|
||||||
|
run "$SECRETS_BIN" add ../escape/.env
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"project-relative"* ]] || false
|
||||||
|
[ ! -f ".secrets.json" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add rejects absolute paths" {
|
||||||
|
create_project_dir addproj
|
||||||
|
run "$SECRETS_BIN" add /etc/passwd
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"project-relative"* ]] || false
|
||||||
|
[ ! -f ".secrets.json" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add rejects shell metacharacters in path" {
|
||||||
|
create_project_dir addproj
|
||||||
|
run "$SECRETS_BIN" add '.env;rm -rf ~'
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[ ! -f ".secrets.json" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "add requires the file to exist" {
|
||||||
|
create_project_dir addproj
|
||||||
|
run "$SECRETS_BIN" add .env.missing
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"not found"* ]] || false
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "manifest serialization is canonical — order of adds does not matter" {
|
||||||
|
create_project_dir addproj
|
||||||
|
echo "A=1" > .env.alpha
|
||||||
|
echo "B=2" > .env.beta
|
||||||
|
"$SECRETS_BIN" add .env.alpha >/dev/null
|
||||||
|
"$SECRETS_BIN" add .env.beta >/dev/null
|
||||||
|
cp .secrets.json "$TEST_TMPDIR/order1.json"
|
||||||
|
rm .secrets.json
|
||||||
|
"$SECRETS_BIN" add .env.beta >/dev/null
|
||||||
|
"$SECRETS_BIN" add .env.alpha >/dev/null
|
||||||
|
cmp -s .secrets.json "$TEST_TMPDIR/order1.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "which shows manifest summary when .secrets.json is present" {
|
||||||
|
create_project_dir addproj
|
||||||
|
"$SECRETS_BIN" add .env >/dev/null
|
||||||
|
run "$SECRETS_BIN" which
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *".secrets.json"* ]] || false
|
||||||
|
[[ "$output" == *".env"* ]] || false
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "malformed .secrets.json dies with a directed error naming the file" {
|
||||||
|
create_project_dir addproj
|
||||||
|
echo '{ not json' > .secrets.json
|
||||||
|
run "$SECRETS_BIN" which
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *".secrets.json"* ]] || false
|
||||||
|
[[ "$output" == *"invalid"* ]] || false
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "unsupported manifest version dies with a directed upgrade error" {
|
||||||
|
create_project_dir addproj
|
||||||
|
echo '{"version": 99, "dotenv": [".env"]}' > .secrets.json
|
||||||
|
run "$SECRETS_BIN" which
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"version 99"* ]] || false
|
||||||
|
[[ "$output" == *"supports"* ]] || false
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "symlinked .secrets.json is refused" {
|
||||||
|
create_project_dir addproj
|
||||||
|
echo '{"version":2,"dotenv":[".env"]}' > "$TEST_TMPDIR/real-manifest.json"
|
||||||
|
ln -s "$TEST_TMPDIR/real-manifest.json" .secrets.json
|
||||||
|
run "$SECRETS_BIN" which
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"symlink"* ]] || false
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue