JC Beasley
6d2105d206
New scripts: - vault-cache.sh: Main cache manager (sync/get/list/status/invalidate) - vault-env.sh: Sourceable environment with quick accessor functions - vault-api-cached.sh: Drop-in replacement for vault_api.sh using cache - nocodb-api.sh: NocoDB API wrapper using cached token - qdrant-api.sh: Qdrant API wrapper using cached key - README-vault-cache.md: Documentation Updated scripts: - vault_grist_api.sh: Now uses vault cache - checks/check-memory-table.sh: Uses vault cache - utils/get-columns.sh: Uses vault cache - utils/update-type-col.sh: Uses vault cache Features: - 29 secrets cached from Vault (6 categories) - Auto-refresh on stale entries (1hr TTL) - Vault token cached for 24h - Restricted permissions (700/600) on cache files - Fallback to direct Vault API if cache miss
401 lines
12 KiB
Bash
Executable File
401 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# vault-cache.sh - Local Vault Secret Cache Manager
|
|
#
|
|
# PURPOSE: Cache Vault secrets locally to avoid repeated Vault API calls
|
|
# - Secrets are encrypted at rest using the Vault approle credentials
|
|
# - Cache auto-refreshes when stale (default: 1 hour TTL)
|
|
# - Supports selective sync, full sync, and cache invalidation
|
|
#
|
|
# USAGE:
|
|
# vault-cache.sh sync # Full sync of all accessible secrets
|
|
# vault-cache.sh sync <category> # Sync specific category (e.g., infrastructure)
|
|
# vault-cache.sh get <path> [field] # Get a specific secret from cache
|
|
# vault-cache.sh list # List cached categories and keys
|
|
# vault-cache.sh invalidate [path|all] # Invalidate cache entry or all
|
|
# vault-cache.sh status # Show cache status and freshness
|
|
#
|
|
# EXAMPLES:
|
|
# vault-cache.sh sync infrastructure # Sync only infrastructure secrets
|
|
# vault-cache.sh get api/data/nocodb token # Get nocodb token
|
|
# vault-cache.sh get api/data/qdrant api-key # Get qdrant API key
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CACHE_DIR="${HOME}/.cache/vault"
|
|
CACHE_DB="${CACHE_DIR}/secrets.db"
|
|
CACHE_META="${CACHE_DIR}/meta.json"
|
|
VAULT_URL="https://beavault.beawit.net:8200"
|
|
ROLE_ID="75d2dcfb-9c65-7f60-59b4-eee8c7f8dc0e"
|
|
SECRET_ID="6202b465-2f25-547c-ec07-f47cfc4dda3e"
|
|
DEFAULT_TTL=3600 # 1 hour in seconds
|
|
|
|
# Ensure cache directory exists with restricted permissions
|
|
init_cache() {
|
|
if [[ ! -d "$CACHE_DIR" ]]; then
|
|
mkdir -p "$CACHE_DIR"
|
|
chmod 700 "$CACHE_DIR"
|
|
echo "Created cache directory: $CACHE_DIR"
|
|
fi
|
|
|
|
# Initialize metadata if doesn't exist
|
|
if [[ ! -f "$CACHE_META" ]]; then
|
|
echo '{"version":1,"created":"'"$(date -Iseconds)"'","entries":{}}' > "$CACHE_META"
|
|
chmod 600 "$CACHE_META"
|
|
fi
|
|
}
|
|
|
|
# Get Vault token (with caching)
|
|
get_vault_token() {
|
|
local token_file="${CACHE_DIR}/.vault_token"
|
|
local token_age=999999
|
|
|
|
if [[ -f "$token_file" ]]; then
|
|
token_age=$(($(date +%s) - $(stat -c %Y "$token_file")))
|
|
fi
|
|
|
|
# Vault tokens are valid for ~32 days, but let's refresh after 24 hours
|
|
if [[ "$token_age" -gt 86400 ]] || [[ ! -s "$token_file" ]]; then
|
|
curl -sk -X POST \
|
|
-d "{\"role_id\":\"${ROLE_ID}\",\"secret_id\":\"${SECRET_ID}\"}" \
|
|
"${VAULT_URL}/v1/auth/approle/login" | \
|
|
jq -r '.auth.client_token' > "$token_file"
|
|
chmod 600 "$token_file"
|
|
fi
|
|
|
|
cat "$token_file"
|
|
}
|
|
|
|
# Fetch a secret from Vault
|
|
fetch_secret() {
|
|
local path="$1"
|
|
local vault_token
|
|
vault_token=$(get_vault_token)
|
|
|
|
local response
|
|
response=$(curl -sk -H "X-Vault-Token: ${vault_token}" \
|
|
"${VAULT_URL}/v1/kv/data/${path}")
|
|
|
|
if echo "$response" | jq -e '.data.data' > /dev/null 2>&1; then
|
|
echo "$response" | jq '.data.data'
|
|
else
|
|
echo "ERROR: Failed to fetch secret at ${path}" >&2
|
|
echo "$response" | jq -r '.errors[]' >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# List secrets in a category
|
|
list_category_secrets() {
|
|
local category="$1"
|
|
local vault_token
|
|
vault_token=$(get_vault_token)
|
|
|
|
curl -sk -H "X-Vault-Token: ${vault_token}" \
|
|
"${VAULT_URL}/v1/kv/metadata/api/${category}?list=true" 2>/dev/null | \
|
|
jq -r '.data.keys[]?' 2>/dev/null || true
|
|
}
|
|
|
|
# Cache a secret to local storage
|
|
cache_secret() {
|
|
local path="$1"
|
|
local data="$2"
|
|
local cache_file="${CACHE_DIR}/$(echo "$path" | tr '/' '_').json"
|
|
|
|
echo "$data" > "$cache_file"
|
|
chmod 600 "$cache_file"
|
|
|
|
# Update metadata
|
|
local meta
|
|
meta=$(jq --arg path "$path" --arg time "$(date +%s)" \
|
|
'.entries[$path] = {"cached_at":$time,"ttl":'$DEFAULT_TTL'}' "$CACHE_META")
|
|
echo "$meta" > "$CACHE_META"
|
|
chmod 600 "$CACHE_META"
|
|
}
|
|
|
|
# Get from cache (returns empty if stale/missing)
|
|
get_cached() {
|
|
local path="$1"
|
|
local cache_file="${CACHE_DIR}/$(echo "$path" | tr '/' '_').json"
|
|
|
|
if [[ ! -f "$cache_file" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
# Check freshness
|
|
local cached_time
|
|
cached_time=$(jq -r --arg path "$path" '.entries[$path].cached_at // 0' "$CACHE_META")
|
|
local age=$(( $(date +%s) - cached_time ))
|
|
local ttl
|
|
ttl=$(jq -r --arg path "$path" '.entries[$path].ttl // '$DEFAULT_TTL'' "$CACHE_META")
|
|
|
|
if [[ "$age" -gt "$ttl" ]]; then
|
|
return 1 # Stale
|
|
fi
|
|
|
|
cat "$cache_file"
|
|
}
|
|
|
|
# Sync a single secret (fetch if stale)
|
|
sync_secret() {
|
|
local path="$1"
|
|
local force="${2:-false}"
|
|
|
|
if [[ "$force" == "true" ]] || ! get_cached "$path" > /dev/null 2>&1; then
|
|
echo " Fetching: ${path}..."
|
|
local data
|
|
if data=$(fetch_secret "$path"); then
|
|
cache_secret "$path" "$data"
|
|
echo " ✓ Cached"
|
|
else
|
|
echo " ✗ Failed"
|
|
return 1
|
|
fi
|
|
else
|
|
echo " Skipping (fresh): ${path}"
|
|
fi
|
|
}
|
|
|
|
# Sync an entire category
|
|
sync_category() {
|
|
local category="$1"
|
|
echo "=== Syncing category: ${category} ==="
|
|
|
|
local secrets
|
|
secrets=$(list_category_secrets "$category")
|
|
|
|
if [[ -z "$secrets" ]]; then
|
|
echo " (empty or no access)"
|
|
return
|
|
fi
|
|
|
|
local count=0
|
|
while IFS= read -r secret; do
|
|
[[ -z "$secret" ]] && continue
|
|
# Remove trailing slash
|
|
secret="${secret%/}"
|
|
sync_secret "api/${category}/${secret}" || true
|
|
count=$((count + 1))
|
|
done <<< "$secrets"
|
|
|
|
echo " Processed ${count} secrets"
|
|
}
|
|
|
|
# Full sync of all categories
|
|
sync_all() {
|
|
echo "=== Full Vault Cache Sync ==="
|
|
echo "Started: $(date)"
|
|
echo ""
|
|
|
|
# Categories we know exist (from discovery)
|
|
local categories=(business communication data infrastructure marketing research)
|
|
|
|
for category in "${categories[@]}"; do
|
|
sync_category "$category"
|
|
echo ""
|
|
done
|
|
|
|
echo "=== Sync Complete ==="
|
|
echo "Finished: $(date)"
|
|
}
|
|
|
|
# Get a value from cache (with auto-fetch if missing)
|
|
get_value() {
|
|
local path="$1"
|
|
local field="${2:-}"
|
|
local force_refresh="${3:-false}"
|
|
|
|
# Auto-fetch if missing or stale
|
|
if [[ "$force_refresh" == "true" ]] || ! get_cached "$path" > /dev/null 2>&1; then
|
|
local data
|
|
if data=$(fetch_secret "$path"); then
|
|
cache_secret "$path" "$data"
|
|
else
|
|
echo "ERROR: Could not retrieve ${path}" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
local cached
|
|
cached=$(get_cached "$path")
|
|
|
|
if [[ -n "$field" ]]; then
|
|
# Try the field name directly, then with underscores instead of hyphens, then bracket notation
|
|
local result
|
|
result=$(echo "$cached" | jq -r ".${field} // empty" 2>/dev/null || true)
|
|
if [[ -z "$result" ]]; then
|
|
# Try with underscores replacing hyphens
|
|
local field_underscore
|
|
field_underscore=$(echo "$field" | tr '-' '_')
|
|
result=$(echo "$cached" | jq -r ".${field_underscore} // empty" 2>/dev/null || true)
|
|
fi
|
|
if [[ -z "$result" ]]; then
|
|
# Try bracket notation for hyphenated keys
|
|
result=$(echo "$cached" | jq -r ".[\"${field}\"] // empty" 2>/dev/null || true)
|
|
fi
|
|
echo "$result"
|
|
else
|
|
echo "$cached" | jq -r 'to_entries | .[] | "\(.key): \(.value)"'
|
|
fi
|
|
}
|
|
|
|
# List cached items
|
|
list_cache() {
|
|
echo "=== Cached Secrets ==="
|
|
|
|
local entries
|
|
entries=$(jq -r '.entries | keys[]' "$CACHE_META" 2>/dev/null)
|
|
|
|
if [[ -z "$entries" ]]; then
|
|
echo " (cache is empty)"
|
|
return
|
|
fi
|
|
|
|
local total=0
|
|
while IFS= read -r path; do
|
|
[[ -z "$path" ]] && continue
|
|
local cached_time
|
|
cached_time=$(jq -r --arg p "$path" '.entries[$p].cached_at // 0' "$CACHE_META")
|
|
local age=$(( $(date +%s) - cached_time ))
|
|
local age_str
|
|
|
|
if [[ $age -lt 60 ]]; then
|
|
age_str="${age}s ago"
|
|
elif [[ $age -lt 3600 ]]; then
|
|
age_str="$((age / 60))m ago"
|
|
else
|
|
age_str="$((age / 3600))h ago"
|
|
fi
|
|
|
|
local ttl
|
|
ttl=$(jq -r --arg p "$path" '.entries[$p].ttl // '$DEFAULT_TTL'' "$CACHE_META")
|
|
local status="✓"
|
|
[[ $age -gt $ttl ]] && status="✗ STALE"
|
|
|
|
printf " %-50s %s %s\n" "$path" "$status" "$age_str"
|
|
total=$((total + 1))
|
|
done <<< "$entries"
|
|
|
|
echo ""
|
|
echo "Total cached: $total"
|
|
}
|
|
|
|
# Show cache status
|
|
show_status() {
|
|
echo "=== Vault Cache Status ==="
|
|
echo "Cache directory: $CACHE_DIR"
|
|
|
|
if [[ -d "$CACHE_DIR" ]]; then
|
|
local disk_usage
|
|
disk_usage=$(du -sh "$CACHE_DIR" 2>/dev/null | cut -f1)
|
|
echo "Disk usage: $disk_usage"
|
|
|
|
local file_count
|
|
file_count=$(find "$CACHE_DIR" -name '*.json' | wc -l)
|
|
echo "Cached secrets: $file_count"
|
|
|
|
# Count stale entries
|
|
local stale_count=0
|
|
local entries
|
|
entries=$(jq -r '.entries | keys[]' "$CACHE_META" 2>/dev/null)
|
|
while IFS= read -r path; do
|
|
[[ -z "$path" ]] && continue
|
|
local cached_time
|
|
cached_time=$(jq -r --arg p "$path" '.entries[$p].cached_at // 0' "$CACHE_META")
|
|
local ttl
|
|
ttl=$(jq -r --arg p "$path" '.entries[$p].ttl // '$DEFAULT_TTL'' "$CACHE_META")
|
|
if [[ $(( $(date +%s) - cached_time )) -gt $ttl ]]; then
|
|
stale_count=$((stale_count + 1))
|
|
fi
|
|
done <<< "$entries"
|
|
|
|
echo "Stale entries: $stale_count"
|
|
else
|
|
echo "Status: Not initialized"
|
|
fi
|
|
|
|
# Vault token status
|
|
local token_file="${CACHE_DIR}/.vault_token"
|
|
if [[ -f "$token_file" ]]; then
|
|
local token_age
|
|
token_age=$(($(date +%s) - $(stat -c %Y "$token_file")))
|
|
echo "Vault token age: $((token_age / 3600))h $(((token_age % 3600) / 60))m"
|
|
else
|
|
echo "Vault token: Not cached"
|
|
fi
|
|
}
|
|
|
|
# Invalidate cache entries
|
|
invalidate_cache() {
|
|
local target="${1:-all}"
|
|
|
|
if [[ "$target" == "all" ]]; then
|
|
rm -f "${CACHE_DIR}"/*.json
|
|
echo '{"version":1,"created":"'"$(date -Iseconds)"'","entries":{}}' > "$CACHE_META"
|
|
echo "Cache fully invalidated"
|
|
else
|
|
local cache_file="${CACHE_DIR}/$(echo "$target" | tr '/' '_').json"
|
|
rm -f "$cache_file"
|
|
|
|
# Update metadata
|
|
local meta
|
|
meta=$(jq --arg path "$target" 'del(.entries[$path])' "$CACHE_META")
|
|
echo "$meta" > "$CACHE_META"
|
|
echo "Invalidated: $target"
|
|
fi
|
|
}
|
|
|
|
# Main command dispatcher
|
|
main() {
|
|
init_cache
|
|
|
|
local cmd="${1:-status}"
|
|
|
|
case "$cmd" in
|
|
sync)
|
|
if [[ -n "${2:-}" ]]; then
|
|
sync_category "$2"
|
|
else
|
|
sync_all
|
|
fi
|
|
;;
|
|
get)
|
|
if [[ -z "${2:-}" ]]; then
|
|
echo "Usage: $0 get <path> [field]"
|
|
exit 1
|
|
fi
|
|
get_value "$2" "${3:-}"
|
|
;;
|
|
list|ls)
|
|
list_cache
|
|
;;
|
|
invalidate|rm|clear)
|
|
invalidate_cache "${2:-all}"
|
|
;;
|
|
status|info)
|
|
show_status
|
|
;;
|
|
*)
|
|
echo "Vault Cache Manager"
|
|
echo ""
|
|
echo "Usage: $0 <command> [args]"
|
|
echo ""
|
|
echo "Commands:"
|
|
echo " sync [category] Sync all or specific category"
|
|
echo " get <path> [field] Get secret from cache (auto-fetch if needed)"
|
|
echo " list List all cached secrets"
|
|
echo " invalidate [path] Invalidate cache (default: all)"
|
|
echo " status Show cache status"
|
|
echo ""
|
|
echo "Examples:"
|
|
echo " $0 sync infrastructure"
|
|
echo " $0 get api/data/nocodb token"
|
|
echo " $0 get api/data/qdrant api-key"
|
|
echo " $0 invalidate api/integration/grist"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|