retention fix
This commit is contained in:
moilanik
@@ -171,3 +171,20 @@ Testit käyttävät `<root>/files/retention-lib.sh` -jaettua kirjastoa, jota my
|
||||
4. Tarkista `TO_DELETE`-array ja `$output`
|
||||
|
||||
**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella).
|
||||
|
||||
---
|
||||
|
||||
## Retention
|
||||
|
||||
Ylläpitoscripti, joka poistaa vanhat raportit git-pagesista retention‑sääntöjen mukaan.
|
||||
Ajetaan sidecar‑ tai cronjob‑tilassa Kubernetesissa.
|
||||
|
||||
### Air gap -yhteensopimattomuus
|
||||
|
||||
Retention‑kontti asentaa tarvitsemansa työkalut (`curl`, `jq`) ajon aikana
|
||||
packagemanagerilla (`apt-get` / `apk`). Tämä **ei toimi air gap -ympäristössä**,
|
||||
jossa konttirekisteriin tai pakettivarastoihin ei ole verkkoyhteyttä.
|
||||
|
||||
**TODO:** Rakenna custom Docker‑image, jossa deps on valmiina:
|
||||
`FROM alpine:latest && apk add --no-cache curl jq`.
|
||||
Pushaa omaan rekisteriin ja päivitä `values.yaml`:n `retention.image`.
|
||||
|
||||
@@ -13,45 +13,8 @@ curl_with_host() {
|
||||
|
||||
[ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }
|
||||
|
||||
declare -A BRANCH_CACHE
|
||||
branch_exists() {
|
||||
local owner="$1" repo="$2" branch="$3" key="${owner}/${repo}/${branch}"
|
||||
local status attempt
|
||||
|
||||
[ -z "$GITEA_API_URL" ] && return 0
|
||||
[ -z "$GITEA_TOKEN" ] && return 0
|
||||
|
||||
if [ "${BRANCH_CACHE[$key]:-}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Retry up to 2 times on API errors (hardcoded)
|
||||
for attempt in 1 2 3; do
|
||||
status=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${GITEA_API_URL}/api/v1/repos/${owner}/${repo}/branches/${branch}" 2>/dev/null || echo "000")
|
||||
|
||||
if [ "$status" = "200" ]; then
|
||||
BRANCH_CACHE[$key]=1
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$status" = "404" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# API error - retry if not last attempt
|
||||
if [ "$attempt" -lt 3 ]; then
|
||||
sleep 10
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
# All retries failed - keep report (fail-safe)
|
||||
echo " WARN: Gitea API error for ${owner}/${repo}/${branch} (status ${status}) after 3 attempts - KEEPING report"
|
||||
BRANCH_CACHE[$key]=1
|
||||
return 0
|
||||
}
|
||||
declare -A REPO_BRANCHES_CACHE
|
||||
declare -A REPO_STATUS
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/retention-lib.sh"
|
||||
@@ -70,6 +33,7 @@ fi
|
||||
echo ""
|
||||
echo "=== Phase 1: collect reports ==="
|
||||
declare -A SEEN_REPORTS
|
||||
declare -A SEEN_ECHO_COMMITS
|
||||
declare -a REPORTS
|
||||
while IFS= read -r meta_path; do
|
||||
report_dir=$(dirname "$meta_path")
|
||||
@@ -90,34 +54,105 @@ while IFS= read -r meta_path; do
|
||||
|
||||
days=$(age_days "$published")
|
||||
REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")
|
||||
echo " ${OWNER}/${REPO} branch=${branch} age=${days}d"
|
||||
|
||||
commit_dir=$(dirname "$report_dir")
|
||||
if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then
|
||||
SEEN_ECHO_COMMITS[$commit_dir]=1
|
||||
echo " ${commit_dir} branch=${branch} age=${days}d"
|
||||
fi
|
||||
done <<< "$META_PATHS"
|
||||
|
||||
[ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }
|
||||
|
||||
echo ""
|
||||
echo "=== Phase 2: check branches in Gitea ==="
|
||||
echo "=== Phase 2: check branches/repos in Gitea ==="
|
||||
if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then
|
||||
echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2
|
||||
exit 1
|
||||
fi
|
||||
declare -a TO_DELETE
|
||||
declare -a KEEP
|
||||
declare -A SEEN_ECHO_BRANCHES
|
||||
declare -A SEEN_ECHO_REPO_DELETED
|
||||
declare -A UNIQUE_BRANCHES
|
||||
declare -A REASON_MAP
|
||||
declare -A COMMIT_BRANCH_MAP
|
||||
|
||||
# Build commit→branch mapping
|
||||
for entry in "${REPORTS[@]}"; do
|
||||
IFS='|' read -r dir _ _ branch _ <<< "$entry"
|
||||
commit_dir=$(dirname "$dir")
|
||||
[ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch
|
||||
done
|
||||
for entry in "${REPORTS[@]}"; do
|
||||
IFS='|' read -r _ owner repo branch _ <<< "$entry"
|
||||
UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1
|
||||
done
|
||||
TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]}
|
||||
BRANCHES_EXISTING=0
|
||||
BRANCH_DELETED_COUNT=0
|
||||
REPO_DELETED_COUNT=0
|
||||
MAXAGE_DELETED=0
|
||||
KEEPMIN_DELETED=0
|
||||
for entry in "${REPORTS[@]}"; do
|
||||
IFS='|' read -r dir owner repo branch days <<< "$entry"
|
||||
|
||||
if [ -n "$GITEA_API_URL" ] && [ -n "$GITEA_TOKEN" ]; then
|
||||
branch_key="${owner}/${repo}/${branch}"
|
||||
if branch_exists "$owner" "$repo" "$branch"; then
|
||||
echo " BRANCH EXISTS: ${owner}/${repo}/${branch}"
|
||||
KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
|
||||
else
|
||||
echo " BRANCH DELETED: ${owner}/${repo}/${branch} -> DELETE"
|
||||
TO_DELETE+=("$dir")
|
||||
if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
|
||||
SEEN_ECHO_BRANCHES[$branch_key]=1
|
||||
BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1))
|
||||
echo " BRANCH EXISTS: ${branch_key}"
|
||||
fi
|
||||
else
|
||||
KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
|
||||
else
|
||||
if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
|
||||
SEEN_ECHO_BRANCHES[$branch_key]=1
|
||||
repo_key="${owner}/${repo}"
|
||||
if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then
|
||||
REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1))
|
||||
if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then
|
||||
SEEN_ECHO_REPO_DELETED[$repo_key]=1
|
||||
echo " REPO DELETED: ${repo_key} -> DELETE ALL"
|
||||
fi
|
||||
reason="repo deleted"
|
||||
else
|
||||
BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1))
|
||||
echo " BRANCH DELETED: ${branch_key} -> DELETE"
|
||||
reason="branch deleted"
|
||||
fi
|
||||
fi
|
||||
REASON_MAP["$dir"]="$reason"
|
||||
TO_DELETE+=("$dir")
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Phase 3: apply retention rules to remaining reports ==="
|
||||
PHASE2_DELETED=${#TO_DELETE[@]}
|
||||
apply_retention "$CONFIG"
|
||||
PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED ))
|
||||
|
||||
fmt_num() {
|
||||
local n="$1" out=""
|
||||
[ -z "$n" ] && { echo "?"; return; }
|
||||
n="${n##0}" # strip leading zeros
|
||||
while [ "${#n}" -gt 3 ]; do
|
||||
out=" ${n: -3}$out"
|
||||
n="${n:0:${#n}-3}"
|
||||
done
|
||||
echo "${n}${out}"
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo " Branches:"
|
||||
echo " existing: $(fmt_num $BRANCHES_EXISTING)"
|
||||
echo " deleted: $(fmt_num $BRANCH_DELETED_COUNT)"
|
||||
echo " repo gone: $(fmt_num $REPO_DELETED_COUNT)"
|
||||
echo " Commits:"
|
||||
echo " deleted by maxAge: $(fmt_num $MAXAGE_DELETED)"
|
||||
echo " deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)"
|
||||
|
||||
if [ "${#TO_DELETE[@]}" -eq 0 ]; then
|
||||
echo "Nothing to delete"
|
||||
@@ -142,14 +177,37 @@ echo "Downloading archive.tar..."
|
||||
HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
|
||||
echo "Extracting archive..."
|
||||
OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}')
|
||||
echo "Extracting archive (${OLD_KB}kB)..."
|
||||
tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"
|
||||
|
||||
for dir in "${TO_DELETE[@]}"; do
|
||||
if [ -d "$SITE_DIR/$dir" ]; then
|
||||
echo " Removing: $dir"
|
||||
rm -rf "$SITE_DIR/$dir"
|
||||
declare -A GROUP_SEEN
|
||||
declare -A GROUP_LINES
|
||||
for del in "${TO_DELETE[@]}"; do
|
||||
if [ ! -d "$SITE_DIR/$del" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
commit_dir=$(dirname "$del")
|
||||
branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}"
|
||||
reason="${REASON_MAP[$del]:-?}"
|
||||
repo_path="${del%%/reports/*}"
|
||||
commit_hash="${commit_dir##*/}"
|
||||
key="${repo_path}/${branch} | Reason: ${reason}"
|
||||
seen_key="${key}|${commit_hash}"
|
||||
|
||||
if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then
|
||||
GROUP_SEEN[$seen_key]=1
|
||||
GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash"
|
||||
fi
|
||||
|
||||
rm -rf "$SITE_DIR/$del"
|
||||
done
|
||||
for key in "${!GROUP_LINES[@]}"; do
|
||||
echo " Removing: ${key}"
|
||||
for hash in ${GROUP_LINES[$key]}; do
|
||||
echo " commit: ${hash}"
|
||||
done
|
||||
done
|
||||
else
|
||||
echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"
|
||||
@@ -197,6 +255,7 @@ if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then
|
||||
fi
|
||||
|
||||
tar -cf "$NEW_TAR" -C "$SITE_DIR" .
|
||||
NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
echo "PUT: replacing site contents..."
|
||||
HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
|
||||
@@ -208,6 +267,9 @@ HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
|
||||
echo "HTTP ${HTTP_CODE}"
|
||||
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
|
||||
echo "Site rebuild completed."
|
||||
if [ -n "${OLD_KB:-}" ]; then
|
||||
echo " archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB"
|
||||
fi
|
||||
else
|
||||
echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
|
||||
exit 1
|
||||
|
||||
@@ -23,6 +23,102 @@ read_rule() {
|
||||
[ -n "$v" ] && echo "$v" || echo "$default"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gitea branch/repo checking via git ls-remote
|
||||
# Uses global: GITEA_API_URL, GITEA_TOKEN
|
||||
# Sets global: REPO_BRANCHES_CACHE, REPO_STATUS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Fetch all branches for a repo (one git ls-remote call per repo).
|
||||
# Sets REPO_STATUS[owner/repo].
|
||||
# Echos branch list on success.
|
||||
# Returns: 0=ok, 1=deleted, 2=cert_error, 3=error (fail-safe keep)
|
||||
repo_branches() {
|
||||
local owner="$1" repo="$2" key="${owner}/${repo}"
|
||||
local attempt output
|
||||
|
||||
[ -z "$GITEA_API_URL" ] && return 0
|
||||
[ -z "$GITEA_TOKEN" ] && return 0
|
||||
|
||||
# Check cached status first (avoids re-running git on every report)
|
||||
case "${REPO_STATUS[$key]:-}" in
|
||||
deleted) return 1 ;;
|
||||
cert_error) return 2 ;;
|
||||
error) echo "${REPO_BRANCHES_CACHE[$key]:-}"; return 3 ;;
|
||||
esac
|
||||
|
||||
# Cache hit (success with branch list)
|
||||
[ -n "${REPO_BRANCHES_CACHE[$key]:-}" ] && { echo "${REPO_BRANCHES_CACHE[$key]}"; return 0; }
|
||||
|
||||
local git_host
|
||||
git_host=$(echo "$GITEA_API_URL" | sed -E 's|^https?://||' | sed 's|/.*$||')
|
||||
local git_url="https://token:${GITEA_TOKEN}@${git_host}/${owner}/${repo}.git"
|
||||
|
||||
for attempt in 1 2 3; do
|
||||
output=$(git ls-remote --heads "$git_url" 2>&1) && {
|
||||
local branches
|
||||
branches=$(echo "$output" | sed -n 's|.*refs/heads/||p')
|
||||
REPO_BRANCHES_CACHE[$key]="$branches"
|
||||
REPO_STATUS[$key]="ok"
|
||||
echo "$branches"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Repo deleted → no retry
|
||||
if echo "$output" | grep -qiE "fatal:.*(not found|repository.*not|could not read)"; then
|
||||
REPO_BRANCHES_CACHE[$key]="__REPO_DELETED__"
|
||||
REPO_STATUS[$key]="deleted"
|
||||
echo " REPO DELETED: ${owner}/${repo}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
[ "$attempt" -lt 3 ] && sleep 10
|
||||
done
|
||||
|
||||
# Certificate verification failure → configuration error, stop
|
||||
if echo "$output" | grep -qi "server certificate verification failed"; then
|
||||
REPO_STATUS[$key]="cert_error"
|
||||
echo "[ERROR] git-pages.retention: certificate verification failed for ${owner}/${repo}" >&2
|
||||
echo "[ERROR] git-pages.retention: check CA certificates or set GIT_SSL_NO_VERIFY=1" >&2
|
||||
echo "[ERROR] git-pages.retention: git output:" >&2
|
||||
echo "$output" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Other network errors → fail-safe keep, continue
|
||||
REPO_BRANCHES_CACHE[$key]="__REPO_ERROR__"
|
||||
REPO_STATUS[$key]="error"
|
||||
echo "[WARN] git-pages.retention: cannot reach Gitea for ${owner}/${repo} — keeping all reports" >&2
|
||||
echo "[WARN] git-pages.retention: git output:" >&2
|
||||
echo "$output" >&2
|
||||
return 3
|
||||
}
|
||||
|
||||
# Check if a specific branch exists in a repo.
|
||||
# Returns 0 (exists), 1 (not found/deleted).
|
||||
# Returns 2 (cert error), 3 (network error).
|
||||
branch_exists() {
|
||||
local owner="$1" repo="$2" branch="$3"
|
||||
local branches rc
|
||||
|
||||
[ -z "$GITEA_API_URL" ] && return 0
|
||||
[ -z "$GITEA_TOKEN" ] && return 0
|
||||
|
||||
branches=$(repo_branches "$owner" "$repo")
|
||||
rc=$?
|
||||
|
||||
# Return codes from repo_branches propagate through $() subshell:
|
||||
# 0=ok, 1=deleted, 2=cert_error, 3=error
|
||||
case $rc in
|
||||
2) echo "[FATAL] git-pages.retention: cannot reach Gitea (${owner}/${repo}) — check configuration" >&2
|
||||
exit 1 ;;
|
||||
3) return 0 ;; # network error → fail-safe keep
|
||||
1) return 1 ;; # repo/branch gone
|
||||
esac
|
||||
|
||||
echo "$branches" | grep -qxF "$branch"
|
||||
}
|
||||
|
||||
# Phase 3: apply retention rules to KEEP array, populate TO_DELETE
|
||||
# Reads from global KEEP array
|
||||
# Populates global TO_DELETE array
|
||||
@@ -30,13 +126,12 @@ read_rule() {
|
||||
apply_retention() {
|
||||
local config="$1"
|
||||
local default_max_age default_keep_min
|
||||
local max_age keep_min key count seen_key commit_dir report_type path
|
||||
local max_age keep_min key count seen_key commit_dir
|
||||
local entry dir owner repo branch days
|
||||
|
||||
default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config")
|
||||
default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config")
|
||||
|
||||
TO_DELETE=()
|
||||
declare -A BRANCH_COUNTS
|
||||
declare -A SEEN_COMMITS
|
||||
declare -A DELETED_COMMITS
|
||||
@@ -56,6 +151,8 @@ apply_retention() {
|
||||
if [ "$days" -gt "$max_age" ]; then
|
||||
echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
|
||||
TO_DELETE+=("$dir")
|
||||
REASON_MAP["$dir"]="maxAgeDays exceed"
|
||||
MAXAGE_DELETED=$((MAXAGE_DELETED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -76,9 +173,10 @@ apply_retention() {
|
||||
if [ "$count" -gt "$keep_min" ]; then
|
||||
if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then
|
||||
DELETED_COMMITS[$commit_dir]=1
|
||||
report_type="${dir##*/}"
|
||||
echo " DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})"
|
||||
TO_DELETE+=("$commit_dir")
|
||||
REASON_MAP["$commit_dir"]="keepMin exceed"
|
||||
KEEPMIN_DELETED=$((KEEPMIN_DELETED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -70,8 +70,15 @@ spec:
|
||||
set -euo pipefail
|
||||
echo "Retention sidecar: installing deps..."
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends curl jq python3 >/dev/null
|
||||
apt-get install -y --no-install-recommends curl jq git ca-certificates >/dev/null
|
||||
echo "Retention sidecar: ready"
|
||||
# Sleep until 01:00 so retention runs at night
|
||||
now_epoch=$(date +%s)
|
||||
target_epoch=$(date -d "today 01:00:00" +%s)
|
||||
[ "$target_epoch" -le "$now_epoch" ] && target_epoch=$((target_epoch + 86400))
|
||||
sleep_sec=$((target_epoch - now_epoch))
|
||||
echo "Retention sidecar: next run in $((sleep_sec / 3600))h (at 01:00)"
|
||||
sleep $sleep_sec
|
||||
while true; do
|
||||
/scripts/retention-cleanup.sh
|
||||
echo "Retention sidecar: next run in 24h"
|
||||
|
||||
@@ -33,7 +33,7 @@ spec:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends curl jq >/dev/null
|
||||
apt-get install -y --no-install-recommends curl jq git >/dev/null
|
||||
chmod +x /scripts/retention-run.sh /scripts/retention-cleanup.sh
|
||||
/scripts/retention-run.sh
|
||||
env:
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
setup() {
|
||||
# Copy the retention lib to a temp location so it's easier to source
|
||||
# (bats runs each test in its own dir)
|
||||
source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh"
|
||||
declare -gA REPO_BRANCHES_CACHE
|
||||
declare -gA REPO_STATUS
|
||||
declare -gA REASON_MAP
|
||||
MAXAGE_DELETED=0
|
||||
KEEPMIN_DELETED=0
|
||||
CONFIG=$(mktemp)
|
||||
}
|
||||
|
||||
@@ -326,3 +329,295 @@ EOF
|
||||
apply_retention "$CONFIG"
|
||||
[ "${#TO_DELETE[@]}" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "TO_DELETE preserves Phase 2 entries after apply_retention (no new deletions)" {
|
||||
write_config <<'EOF'
|
||||
{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}}
|
||||
EOF
|
||||
|
||||
KEEP=(
|
||||
"niko/r/reports/c1/test|niko|r|main|10"
|
||||
"niko/r/reports/c2/test|niko|r|main|5"
|
||||
)
|
||||
TO_DELETE=(
|
||||
"niko/r/reports/abc/branch-gone"
|
||||
"niko/r/reports/def/repo-gone"
|
||||
)
|
||||
|
||||
apply_retention "$CONFIG"
|
||||
|
||||
[ "${#TO_DELETE[@]}" -eq 2 ]
|
||||
[[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]]
|
||||
[[ "${TO_DELETE[1]}" == "niko/r/reports/def/repo-gone" ]]
|
||||
}
|
||||
|
||||
@test "TO_DELETE preserves Phase 2 entries AND adds retention deletions" {
|
||||
write_config <<'EOF'
|
||||
{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}}
|
||||
EOF
|
||||
|
||||
KEEP=(
|
||||
"niko/r/reports/c1/test|niko|r|main|40"
|
||||
"niko/r/reports/c2/test|niko|r|main|30"
|
||||
"niko/r/reports/c3/test|niko|r|main|20"
|
||||
"niko/r/reports/c4/test|niko|r|main|10"
|
||||
)
|
||||
TO_DELETE=(
|
||||
"niko/r/reports/abc/branch-gone"
|
||||
)
|
||||
|
||||
apply_retention "$CONFIG"
|
||||
|
||||
# 1 pre-existing + 1 commit deleted (c1, oldest of 4, keepMin=3)
|
||||
[ "${#TO_DELETE[@]}" -eq 2 ]
|
||||
[[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]]
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# branch_exists — mocking REPO_STATUS / REPO_BRANCHES_CACHE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "branch_exists: branch in list → return 0" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test-token"
|
||||
|
||||
REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x'
|
||||
REPO_STATUS["owner/repo"]="ok"
|
||||
|
||||
run branch_exists "owner" "repo" "main"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "branch_exists: branch not in list → return 1" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test-token"
|
||||
|
||||
REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x'
|
||||
REPO_STATUS["owner/repo"]="ok"
|
||||
|
||||
run branch_exists "owner" "repo" "nonexistent"
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "branch_exists: cert error → exit 1 with [FATAL]" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test"
|
||||
|
||||
REPO_STATUS["owner/repo"]="cert_error"
|
||||
|
||||
run branch_exists "owner" "repo" "any-branch"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"[FATAL]"* ]]
|
||||
}
|
||||
|
||||
@test "branch_exists: repo deleted → return 1" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test-token"
|
||||
|
||||
REPO_BRANCHES_CACHE["owner/repo"]="__REPO_DELETED__"
|
||||
REPO_STATUS["owner/repo"]="deleted"
|
||||
|
||||
run branch_exists "owner" "repo" "any-branch"
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "branch_exists: network error → return 0 (fail-safe keep)" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test-token"
|
||||
|
||||
REPO_BRANCHES_CACHE["owner/repo"]="__REPO_ERROR__"
|
||||
REPO_STATUS["owner/repo"]="error"
|
||||
|
||||
run branch_exists "owner" "repo" "any-branch"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "branch_exists: empty GITEA_API_URL → return 0 (skip)" {
|
||||
GITEA_API_URL=""
|
||||
GITEA_TOKEN="test-token"
|
||||
|
||||
run branch_exists "owner" "repo" "any-branch"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "branch_exists: empty GITEA_TOKEN → return 0 (skip)" {
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN=""
|
||||
|
||||
run branch_exists "owner" "repo" "any-branch"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# repo_branches — git ls-remote error detection patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "error detection: 'command not found' does NOT trigger repo deleted" {
|
||||
# This must NOT match — "bash: git: command not found" is NOT a repo deletion
|
||||
local msg="bash: git: command not found"
|
||||
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "error detection: 'fatal: repo not found' triggers repo deleted" {
|
||||
# This MUST match — genuine git error for deleted/missing repo
|
||||
local msg="fatal: repository 'https://gitea.app/owner/repo.git' not found"
|
||||
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "error detection: 'could not read from remote' triggers repo deleted" {
|
||||
local msg="fatal: could not read from remote repository"
|
||||
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# git ls-remote integration (real git, temp repo)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "git ls-remote parsing: lists branches correctly" {
|
||||
local tmpdir=$(mktemp -d)
|
||||
|
||||
git -C "$tmpdir" init -b main source >/dev/null 2>&1
|
||||
git -C "$tmpdir/source" config user.email "test@test"
|
||||
git -C "$tmpdir/source" config user.name "test"
|
||||
git -C "$tmpdir/source" commit --allow-empty -m "init" >/dev/null 2>&1
|
||||
git -C "$tmpdir/source" branch feature/x >/dev/null 2>&1
|
||||
git clone --bare "$tmpdir/source" "$tmpdir/repo.git" >/dev/null 2>&1
|
||||
|
||||
local url="file://$tmpdir/repo.git"
|
||||
local output
|
||||
output=$(git ls-remote --heads "$url" 2>&1)
|
||||
local branches
|
||||
branches=$(echo "$output" | sed -n 's|.*refs/heads/||p')
|
||||
|
||||
echo "$branches" | grep -qxF "main"
|
||||
[ "$?" -eq 0 ]
|
||||
|
||||
echo "$branches" | grep -qxF "feature/x"
|
||||
[ "$?" -eq 0 ]
|
||||
|
||||
! echo "$branches" | grep -qxF "nonexistent"
|
||||
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# repo_branches — retry + error output (using git mock)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "repo_branches: success returns branches immediately" {
|
||||
local mockdir=$(mktemp -d)
|
||||
cat > "$mockdir/git" << 'SCRIPT'
|
||||
#!/usr/bin/env bash
|
||||
echo "abc123 refs/heads/main"
|
||||
echo "def456 refs/heads/feature/x"
|
||||
SCRIPT
|
||||
chmod +x "$mockdir/git"
|
||||
local save_PATH="$PATH"
|
||||
export PATH="$mockdir:$PATH"
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test"
|
||||
REPO_BRANCHES_CACHE=()
|
||||
REPO_STATUS=()
|
||||
|
||||
run repo_branches "owner" "repo"
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"main"* ]]
|
||||
[[ "$output" == *"feature/x"* ]]
|
||||
|
||||
export PATH="$save_PATH"
|
||||
rm -rf "$mockdir"
|
||||
}
|
||||
|
||||
@test "repo_branches: retries 3 times on transient error" {
|
||||
local mockdir=$(mktemp -d)
|
||||
cat > "$mockdir/git" << 'SCRIPT'
|
||||
#!/usr/bin/env bash
|
||||
echo "call" >> "$MOCKDIR/count"
|
||||
echo "fatal: unable to access 'https://...'" >&2
|
||||
exit 1
|
||||
SCRIPT
|
||||
chmod +x "$mockdir/git"
|
||||
# Inject mockdir path into mock script via env var
|
||||
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||
local save_PATH="$PATH"
|
||||
export PATH="$mockdir:$PATH"
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test"
|
||||
REPO_BRANCHES_CACHE=()
|
||||
REPO_STATUS=()
|
||||
|
||||
local start=$SECONDS
|
||||
run repo_branches "owner" "repo"
|
||||
|
||||
[ "$status" -eq 3 ]
|
||||
[[ "$output" == *"[WARN] git-pages.retention"* ]]
|
||||
[[ "$output" == *"keeping all reports"* ]]
|
||||
[[ "$output" == *"git output:"* ]]
|
||||
[[ "$output" == *"unable to access"* ]]
|
||||
[ $(cat "$mockdir/count" | wc -l) -eq 3 ]
|
||||
[ $(( SECONDS - start )) -ge 18 ]
|
||||
|
||||
export PATH="$save_PATH"
|
||||
rm -rf "$mockdir"
|
||||
}
|
||||
|
||||
@test "repo_branches: certificate error → [ERROR] + return 1" {
|
||||
local mockdir=$(mktemp -d)
|
||||
cat > "$mockdir/git" << 'SCRIPT'
|
||||
#!/usr/bin/env bash
|
||||
echo "call" >> "$MOCKDIR/count"
|
||||
echo "fatal: unable to access 'https://gitea.app/owner/repo.git/': server certificate verification failed. CAfile: none CRLfile: none" >&2
|
||||
exit 1
|
||||
SCRIPT
|
||||
chmod +x "$mockdir/git"
|
||||
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||
local save_PATH="$PATH"
|
||||
export PATH="$mockdir:$PATH"
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test"
|
||||
REPO_BRANCHES_CACHE=()
|
||||
REPO_STATUS=()
|
||||
|
||||
run repo_branches "owner" "repo"
|
||||
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"[ERROR]"* ]]
|
||||
[[ "$output" == *"certificate verification"* ]]
|
||||
[[ "$output" == *"git output:"* ]]
|
||||
[[ "$output" == *"unable to access"* ]]
|
||||
|
||||
export PATH="$save_PATH"
|
||||
rm -rf "$mockdir"
|
||||
}
|
||||
|
||||
@test "repo_branches: repo not found returns immediately (no retry)" {
|
||||
local mockdir=$(mktemp -d)
|
||||
cat > "$mockdir/git" << 'SCRIPT'
|
||||
#!/usr/bin/env bash
|
||||
echo "call" >> "$MOCKDIR/count"
|
||||
echo "fatal: repository 'https://gitea.app/owner/repo.git' not found" >&2
|
||||
exit 1
|
||||
SCRIPT
|
||||
chmod +x "$mockdir/git"
|
||||
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||
local save_PATH="$PATH"
|
||||
export PATH="$mockdir:$PATH"
|
||||
GITEA_API_URL="https://gitea.example.com"
|
||||
GITEA_TOKEN="test"
|
||||
REPO_BRANCHES_CACHE=()
|
||||
REPO_STATUS=()
|
||||
|
||||
run repo_branches "owner" "repo"
|
||||
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"REPO DELETED"* ]]
|
||||
[[ "$output" != *"[WARN]"* ]]
|
||||
[ $(cat "$mockdir/count" | wc -l) -eq 1 ]
|
||||
|
||||
export PATH="$save_PATH"
|
||||
rm -rf "$mockdir"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user