retention fix
This commit is contained in:
@@ -171,3 +171,20 @@ Testit käyttävät `<root>/files/retention-lib.sh` -jaettua kirjastoa, jota my
|
|||||||
4. Tarkista `TO_DELETE`-array ja `$output`
|
4. Tarkista `TO_DELETE`-array ja `$output`
|
||||||
|
|
||||||
**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella).
|
**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Retention
|
||||||
|
|
||||||
|
Ylläpitoscripti, joka poistaa vanhat raportit git-pagesista retention‑sääntöjen mukaan.
|
||||||
|
Ajetaan sidecar‑ tai cronjob‑tilassa Kubernetesissa.
|
||||||
|
|
||||||
|
### Air gap -yhteensopimattomuus
|
||||||
|
|
||||||
|
Retention‑kontti asentaa tarvitsemansa työkalut (`curl`, `jq`) ajon aikana
|
||||||
|
packagemanagerilla (`apt-get` / `apk`). Tämä **ei toimi air gap -ympäristössä**,
|
||||||
|
jossa konttirekisteriin tai pakettivarastoihin ei ole verkkoyhteyttä.
|
||||||
|
|
||||||
|
**TODO:** Rakenna custom Docker‑image, jossa deps on valmiina:
|
||||||
|
`FROM alpine:latest && apk add --no-cache curl jq`.
|
||||||
|
Pushaa omaan rekisteriin ja päivitä `values.yaml`:n `retention.image`.
|
||||||
|
|||||||
@@ -13,45 +13,8 @@ curl_with_host() {
|
|||||||
|
|
||||||
[ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }
|
[ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }
|
||||||
|
|
||||||
declare -A BRANCH_CACHE
|
declare -A REPO_BRANCHES_CACHE
|
||||||
branch_exists() {
|
declare -A REPO_STATUS
|
||||||
local owner="$1" repo="$2" branch="$3" key="${owner}/${repo}/${branch}"
|
|
||||||
local status attempt
|
|
||||||
|
|
||||||
[ -z "$GITEA_API_URL" ] && return 0
|
|
||||||
[ -z "$GITEA_TOKEN" ] && return 0
|
|
||||||
|
|
||||||
if [ "${BRANCH_CACHE[$key]:-}" = "1" ]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Retry up to 2 times on API errors (hardcoded)
|
|
||||||
for attempt in 1 2 3; do
|
|
||||||
status=$(curl -sS -o /dev/null -w "%{http_code}" \
|
|
||||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
|
||||||
"${GITEA_API_URL}/api/v1/repos/${owner}/${repo}/branches/${branch}" 2>/dev/null || echo "000")
|
|
||||||
|
|
||||||
if [ "$status" = "200" ]; then
|
|
||||||
BRANCH_CACHE[$key]=1
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$status" = "404" ]; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# API error - retry if not last attempt
|
|
||||||
if [ "$attempt" -lt 3 ]; then
|
|
||||||
sleep 10
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# All retries failed - keep report (fail-safe)
|
|
||||||
echo " WARN: Gitea API error for ${owner}/${repo}/${branch} (status ${status}) after 3 attempts - KEEPING report"
|
|
||||||
BRANCH_CACHE[$key]=1
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/retention-lib.sh"
|
source "$SCRIPT_DIR/retention-lib.sh"
|
||||||
@@ -70,6 +33,7 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "=== Phase 1: collect reports ==="
|
echo "=== Phase 1: collect reports ==="
|
||||||
declare -A SEEN_REPORTS
|
declare -A SEEN_REPORTS
|
||||||
|
declare -A SEEN_ECHO_COMMITS
|
||||||
declare -a REPORTS
|
declare -a REPORTS
|
||||||
while IFS= read -r meta_path; do
|
while IFS= read -r meta_path; do
|
||||||
report_dir=$(dirname "$meta_path")
|
report_dir=$(dirname "$meta_path")
|
||||||
@@ -90,34 +54,105 @@ while IFS= read -r meta_path; do
|
|||||||
|
|
||||||
days=$(age_days "$published")
|
days=$(age_days "$published")
|
||||||
REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")
|
REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")
|
||||||
echo " ${OWNER}/${REPO} branch=${branch} age=${days}d"
|
|
||||||
|
commit_dir=$(dirname "$report_dir")
|
||||||
|
if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then
|
||||||
|
SEEN_ECHO_COMMITS[$commit_dir]=1
|
||||||
|
echo " ${commit_dir} branch=${branch} age=${days}d"
|
||||||
|
fi
|
||||||
done <<< "$META_PATHS"
|
done <<< "$META_PATHS"
|
||||||
|
|
||||||
[ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }
|
[ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Phase 2: check branches in Gitea ==="
|
echo "=== Phase 2: check branches/repos in Gitea ==="
|
||||||
|
if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then
|
||||||
|
echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
declare -a TO_DELETE
|
declare -a TO_DELETE
|
||||||
declare -a KEEP
|
declare -a KEEP
|
||||||
|
declare -A SEEN_ECHO_BRANCHES
|
||||||
|
declare -A SEEN_ECHO_REPO_DELETED
|
||||||
|
declare -A UNIQUE_BRANCHES
|
||||||
|
declare -A REASON_MAP
|
||||||
|
declare -A COMMIT_BRANCH_MAP
|
||||||
|
|
||||||
|
# Build commit→branch mapping
|
||||||
|
for entry in "${REPORTS[@]}"; do
|
||||||
|
IFS='|' read -r dir _ _ branch _ <<< "$entry"
|
||||||
|
commit_dir=$(dirname "$dir")
|
||||||
|
[ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch
|
||||||
|
done
|
||||||
|
for entry in "${REPORTS[@]}"; do
|
||||||
|
IFS='|' read -r _ owner repo branch _ <<< "$entry"
|
||||||
|
UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1
|
||||||
|
done
|
||||||
|
TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]}
|
||||||
|
BRANCHES_EXISTING=0
|
||||||
|
BRANCH_DELETED_COUNT=0
|
||||||
|
REPO_DELETED_COUNT=0
|
||||||
|
MAXAGE_DELETED=0
|
||||||
|
KEEPMIN_DELETED=0
|
||||||
for entry in "${REPORTS[@]}"; do
|
for entry in "${REPORTS[@]}"; do
|
||||||
IFS='|' read -r dir owner repo branch days <<< "$entry"
|
IFS='|' read -r dir owner repo branch days <<< "$entry"
|
||||||
|
|
||||||
if [ -n "$GITEA_API_URL" ] && [ -n "$GITEA_TOKEN" ]; then
|
branch_key="${owner}/${repo}/${branch}"
|
||||||
if branch_exists "$owner" "$repo" "$branch"; then
|
if branch_exists "$owner" "$repo" "$branch"; then
|
||||||
echo " BRANCH EXISTS: ${owner}/${repo}/${branch}"
|
if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
|
||||||
KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
|
SEEN_ECHO_BRANCHES[$branch_key]=1
|
||||||
else
|
BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1))
|
||||||
echo " BRANCH DELETED: ${owner}/${repo}/${branch} -> DELETE"
|
echo " BRANCH EXISTS: ${branch_key}"
|
||||||
TO_DELETE+=("$dir")
|
|
||||||
fi
|
fi
|
||||||
else
|
|
||||||
KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
|
KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
|
||||||
|
else
|
||||||
|
if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
|
||||||
|
SEEN_ECHO_BRANCHES[$branch_key]=1
|
||||||
|
repo_key="${owner}/${repo}"
|
||||||
|
if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then
|
||||||
|
REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1))
|
||||||
|
if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then
|
||||||
|
SEEN_ECHO_REPO_DELETED[$repo_key]=1
|
||||||
|
echo " REPO DELETED: ${repo_key} -> DELETE ALL"
|
||||||
|
fi
|
||||||
|
reason="repo deleted"
|
||||||
|
else
|
||||||
|
BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1))
|
||||||
|
echo " BRANCH DELETED: ${branch_key} -> DELETE"
|
||||||
|
reason="branch deleted"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
REASON_MAP["$dir"]="$reason"
|
||||||
|
TO_DELETE+=("$dir")
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Phase 3: apply retention rules to remaining reports ==="
|
echo "=== Phase 3: apply retention rules to remaining reports ==="
|
||||||
|
PHASE2_DELETED=${#TO_DELETE[@]}
|
||||||
apply_retention "$CONFIG"
|
apply_retention "$CONFIG"
|
||||||
|
PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED ))
|
||||||
|
|
||||||
|
fmt_num() {
|
||||||
|
local n="$1" out=""
|
||||||
|
[ -z "$n" ] && { echo "?"; return; }
|
||||||
|
n="${n##0}" # strip leading zeros
|
||||||
|
while [ "${#n}" -gt 3 ]; do
|
||||||
|
out=" ${n: -3}$out"
|
||||||
|
n="${n:0:${#n}-3}"
|
||||||
|
done
|
||||||
|
echo "${n}${out}"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
echo " Branches:"
|
||||||
|
echo " existing: $(fmt_num $BRANCHES_EXISTING)"
|
||||||
|
echo " deleted: $(fmt_num $BRANCH_DELETED_COUNT)"
|
||||||
|
echo " repo gone: $(fmt_num $REPO_DELETED_COUNT)"
|
||||||
|
echo " Commits:"
|
||||||
|
echo " deleted by maxAge: $(fmt_num $MAXAGE_DELETED)"
|
||||||
|
echo " deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)"
|
||||||
|
|
||||||
if [ "${#TO_DELETE[@]}" -eq 0 ]; then
|
if [ "${#TO_DELETE[@]}" -eq 0 ]; then
|
||||||
echo "Nothing to delete"
|
echo "Nothing to delete"
|
||||||
@@ -142,14 +177,37 @@ echo "Downloading archive.tar..."
|
|||||||
HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")
|
HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")
|
||||||
|
|
||||||
if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
|
if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
|
||||||
echo "Extracting archive..."
|
OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}')
|
||||||
|
echo "Extracting archive (${OLD_KB}kB)..."
|
||||||
tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"
|
tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"
|
||||||
|
|
||||||
for dir in "${TO_DELETE[@]}"; do
|
declare -A GROUP_SEEN
|
||||||
if [ -d "$SITE_DIR/$dir" ]; then
|
declare -A GROUP_LINES
|
||||||
echo " Removing: $dir"
|
for del in "${TO_DELETE[@]}"; do
|
||||||
rm -rf "$SITE_DIR/$dir"
|
if [ ! -d "$SITE_DIR/$del" ]; then
|
||||||
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
commit_dir=$(dirname "$del")
|
||||||
|
branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}"
|
||||||
|
reason="${REASON_MAP[$del]:-?}"
|
||||||
|
repo_path="${del%%/reports/*}"
|
||||||
|
commit_hash="${commit_dir##*/}"
|
||||||
|
key="${repo_path}/${branch} | Reason: ${reason}"
|
||||||
|
seen_key="${key}|${commit_hash}"
|
||||||
|
|
||||||
|
if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then
|
||||||
|
GROUP_SEEN[$seen_key]=1
|
||||||
|
GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash"
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf "$SITE_DIR/$del"
|
||||||
|
done
|
||||||
|
for key in "${!GROUP_LINES[@]}"; do
|
||||||
|
echo " Removing: ${key}"
|
||||||
|
for hash in ${GROUP_LINES[$key]}; do
|
||||||
|
echo " commit: ${hash}"
|
||||||
|
done
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"
|
echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"
|
||||||
@@ -197,6 +255,7 @@ if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
tar -cf "$NEW_TAR" -C "$SITE_DIR" .
|
tar -cf "$NEW_TAR" -C "$SITE_DIR" .
|
||||||
|
NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}')
|
||||||
|
|
||||||
echo "PUT: replacing site contents..."
|
echo "PUT: replacing site contents..."
|
||||||
HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
|
HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
|
||||||
@@ -208,6 +267,9 @@ HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
|
|||||||
echo "HTTP ${HTTP_CODE}"
|
echo "HTTP ${HTTP_CODE}"
|
||||||
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
|
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
|
||||||
echo "Site rebuild completed."
|
echo "Site rebuild completed."
|
||||||
|
if [ -n "${OLD_KB:-}" ]; then
|
||||||
|
echo " archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
|
echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -23,6 +23,102 @@ read_rule() {
|
|||||||
[ -n "$v" ] && echo "$v" || echo "$default"
|
[ -n "$v" ] && echo "$v" || echo "$default"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Gitea branch/repo checking via git ls-remote
|
||||||
|
# Uses global: GITEA_API_URL, GITEA_TOKEN
|
||||||
|
# Sets global: REPO_BRANCHES_CACHE, REPO_STATUS
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Fetch all branches for a repo (one git ls-remote call per repo).
|
||||||
|
# Sets REPO_STATUS[owner/repo].
|
||||||
|
# Echos branch list on success.
|
||||||
|
# Returns: 0=ok, 1=deleted, 2=cert_error, 3=error (fail-safe keep)
|
||||||
|
repo_branches() {
|
||||||
|
local owner="$1" repo="$2" key="${owner}/${repo}"
|
||||||
|
local attempt output
|
||||||
|
|
||||||
|
[ -z "$GITEA_API_URL" ] && return 0
|
||||||
|
[ -z "$GITEA_TOKEN" ] && return 0
|
||||||
|
|
||||||
|
# Check cached status first (avoids re-running git on every report)
|
||||||
|
case "${REPO_STATUS[$key]:-}" in
|
||||||
|
deleted) return 1 ;;
|
||||||
|
cert_error) return 2 ;;
|
||||||
|
error) echo "${REPO_BRANCHES_CACHE[$key]:-}"; return 3 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Cache hit (success with branch list)
|
||||||
|
[ -n "${REPO_BRANCHES_CACHE[$key]:-}" ] && { echo "${REPO_BRANCHES_CACHE[$key]}"; return 0; }
|
||||||
|
|
||||||
|
local git_host
|
||||||
|
git_host=$(echo "$GITEA_API_URL" | sed -E 's|^https?://||' | sed 's|/.*$||')
|
||||||
|
local git_url="https://token:${GITEA_TOKEN}@${git_host}/${owner}/${repo}.git"
|
||||||
|
|
||||||
|
for attempt in 1 2 3; do
|
||||||
|
output=$(git ls-remote --heads "$git_url" 2>&1) && {
|
||||||
|
local branches
|
||||||
|
branches=$(echo "$output" | sed -n 's|.*refs/heads/||p')
|
||||||
|
REPO_BRANCHES_CACHE[$key]="$branches"
|
||||||
|
REPO_STATUS[$key]="ok"
|
||||||
|
echo "$branches"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Repo deleted → no retry
|
||||||
|
if echo "$output" | grep -qiE "fatal:.*(not found|repository.*not|could not read)"; then
|
||||||
|
REPO_BRANCHES_CACHE[$key]="__REPO_DELETED__"
|
||||||
|
REPO_STATUS[$key]="deleted"
|
||||||
|
echo " REPO DELETED: ${owner}/${repo}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ "$attempt" -lt 3 ] && sleep 10
|
||||||
|
done
|
||||||
|
|
||||||
|
# Certificate verification failure → configuration error, stop
|
||||||
|
if echo "$output" | grep -qi "server certificate verification failed"; then
|
||||||
|
REPO_STATUS[$key]="cert_error"
|
||||||
|
echo "[ERROR] git-pages.retention: certificate verification failed for ${owner}/${repo}" >&2
|
||||||
|
echo "[ERROR] git-pages.retention: check CA certificates or set GIT_SSL_NO_VERIFY=1" >&2
|
||||||
|
echo "[ERROR] git-pages.retention: git output:" >&2
|
||||||
|
echo "$output" >&2
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Other network errors → fail-safe keep, continue
|
||||||
|
REPO_BRANCHES_CACHE[$key]="__REPO_ERROR__"
|
||||||
|
REPO_STATUS[$key]="error"
|
||||||
|
echo "[WARN] git-pages.retention: cannot reach Gitea for ${owner}/${repo} — keeping all reports" >&2
|
||||||
|
echo "[WARN] git-pages.retention: git output:" >&2
|
||||||
|
echo "$output" >&2
|
||||||
|
return 3
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check if a specific branch exists in a repo.
|
||||||
|
# Returns 0 (exists), 1 (not found/deleted).
|
||||||
|
# Returns 2 (cert error), 3 (network error).
|
||||||
|
branch_exists() {
|
||||||
|
local owner="$1" repo="$2" branch="$3"
|
||||||
|
local branches rc
|
||||||
|
|
||||||
|
[ -z "$GITEA_API_URL" ] && return 0
|
||||||
|
[ -z "$GITEA_TOKEN" ] && return 0
|
||||||
|
|
||||||
|
branches=$(repo_branches "$owner" "$repo")
|
||||||
|
rc=$?
|
||||||
|
|
||||||
|
# Return codes from repo_branches propagate through $() subshell:
|
||||||
|
# 0=ok, 1=deleted, 2=cert_error, 3=error
|
||||||
|
case $rc in
|
||||||
|
2) echo "[FATAL] git-pages.retention: cannot reach Gitea (${owner}/${repo}) — check configuration" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
3) return 0 ;; # network error → fail-safe keep
|
||||||
|
1) return 1 ;; # repo/branch gone
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "$branches" | grep -qxF "$branch"
|
||||||
|
}
|
||||||
|
|
||||||
# Phase 3: apply retention rules to KEEP array, populate TO_DELETE
|
# Phase 3: apply retention rules to KEEP array, populate TO_DELETE
|
||||||
# Reads from global KEEP array
|
# Reads from global KEEP array
|
||||||
# Populates global TO_DELETE array
|
# Populates global TO_DELETE array
|
||||||
@@ -30,13 +126,12 @@ read_rule() {
|
|||||||
apply_retention() {
|
apply_retention() {
|
||||||
local config="$1"
|
local config="$1"
|
||||||
local default_max_age default_keep_min
|
local default_max_age default_keep_min
|
||||||
local max_age keep_min key count seen_key commit_dir report_type path
|
local max_age keep_min key count seen_key commit_dir
|
||||||
local entry dir owner repo branch days
|
local entry dir owner repo branch days
|
||||||
|
|
||||||
default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config")
|
default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config")
|
||||||
default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config")
|
default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config")
|
||||||
|
|
||||||
TO_DELETE=()
|
|
||||||
declare -A BRANCH_COUNTS
|
declare -A BRANCH_COUNTS
|
||||||
declare -A SEEN_COMMITS
|
declare -A SEEN_COMMITS
|
||||||
declare -A DELETED_COMMITS
|
declare -A DELETED_COMMITS
|
||||||
@@ -56,6 +151,8 @@ apply_retention() {
|
|||||||
if [ "$days" -gt "$max_age" ]; then
|
if [ "$days" -gt "$max_age" ]; then
|
||||||
echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
|
echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
|
||||||
TO_DELETE+=("$dir")
|
TO_DELETE+=("$dir")
|
||||||
|
REASON_MAP["$dir"]="maxAgeDays exceed"
|
||||||
|
MAXAGE_DELETED=$((MAXAGE_DELETED + 1))
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -76,9 +173,10 @@ apply_retention() {
|
|||||||
if [ "$count" -gt "$keep_min" ]; then
|
if [ "$count" -gt "$keep_min" ]; then
|
||||||
if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then
|
if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then
|
||||||
DELETED_COMMITS[$commit_dir]=1
|
DELETED_COMMITS[$commit_dir]=1
|
||||||
report_type="${dir##*/}"
|
|
||||||
echo " DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})"
|
echo " DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})"
|
||||||
TO_DELETE+=("$commit_dir")
|
TO_DELETE+=("$commit_dir")
|
||||||
|
REASON_MAP["$commit_dir"]="keepMin exceed"
|
||||||
|
KEEPMIN_DELETED=$((KEEPMIN_DELETED + 1))
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -70,8 +70,15 @@ spec:
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
echo "Retention sidecar: installing deps..."
|
echo "Retention sidecar: installing deps..."
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y --no-install-recommends curl jq python3 >/dev/null
|
apt-get install -y --no-install-recommends curl jq git ca-certificates >/dev/null
|
||||||
echo "Retention sidecar: ready"
|
echo "Retention sidecar: ready"
|
||||||
|
# Sleep until 01:00 so retention runs at night
|
||||||
|
now_epoch=$(date +%s)
|
||||||
|
target_epoch=$(date -d "today 01:00:00" +%s)
|
||||||
|
[ "$target_epoch" -le "$now_epoch" ] && target_epoch=$((target_epoch + 86400))
|
||||||
|
sleep_sec=$((target_epoch - now_epoch))
|
||||||
|
echo "Retention sidecar: next run in $((sleep_sec / 3600))h (at 01:00)"
|
||||||
|
sleep $sleep_sec
|
||||||
while true; do
|
while true; do
|
||||||
/scripts/retention-cleanup.sh
|
/scripts/retention-cleanup.sh
|
||||||
echo "Retention sidecar: next run in 24h"
|
echo "Retention sidecar: next run in 24h"
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ spec:
|
|||||||
- |
|
- |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y --no-install-recommends curl jq >/dev/null
|
apt-get install -y --no-install-recommends curl jq git >/dev/null
|
||||||
chmod +x /scripts/retention-run.sh /scripts/retention-cleanup.sh
|
chmod +x /scripts/retention-run.sh /scripts/retention-cleanup.sh
|
||||||
/scripts/retention-run.sh
|
/scripts/retention-run.sh
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
#!/usr/bin/env bats
|
#!/usr/bin/env bats
|
||||||
|
|
||||||
setup() {
|
setup() {
|
||||||
# Copy the retention lib to a temp location so it's easier to source
|
|
||||||
# (bats runs each test in its own dir)
|
|
||||||
source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh"
|
source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh"
|
||||||
|
declare -gA REPO_BRANCHES_CACHE
|
||||||
|
declare -gA REPO_STATUS
|
||||||
|
declare -gA REASON_MAP
|
||||||
|
MAXAGE_DELETED=0
|
||||||
|
KEEPMIN_DELETED=0
|
||||||
CONFIG=$(mktemp)
|
CONFIG=$(mktemp)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -326,3 +329,295 @@ EOF
|
|||||||
apply_retention "$CONFIG"
|
apply_retention "$CONFIG"
|
||||||
[ "${#TO_DELETE[@]}" -eq 0 ]
|
[ "${#TO_DELETE[@]}" -eq 0 ]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "TO_DELETE preserves Phase 2 entries after apply_retention (no new deletions)" {
|
||||||
|
write_config <<'EOF'
|
||||||
|
{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
KEEP=(
|
||||||
|
"niko/r/reports/c1/test|niko|r|main|10"
|
||||||
|
"niko/r/reports/c2/test|niko|r|main|5"
|
||||||
|
)
|
||||||
|
TO_DELETE=(
|
||||||
|
"niko/r/reports/abc/branch-gone"
|
||||||
|
"niko/r/reports/def/repo-gone"
|
||||||
|
)
|
||||||
|
|
||||||
|
apply_retention "$CONFIG"
|
||||||
|
|
||||||
|
[ "${#TO_DELETE[@]}" -eq 2 ]
|
||||||
|
[[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]]
|
||||||
|
[[ "${TO_DELETE[1]}" == "niko/r/reports/def/repo-gone" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "TO_DELETE preserves Phase 2 entries AND adds retention deletions" {
|
||||||
|
write_config <<'EOF'
|
||||||
|
{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
KEEP=(
|
||||||
|
"niko/r/reports/c1/test|niko|r|main|40"
|
||||||
|
"niko/r/reports/c2/test|niko|r|main|30"
|
||||||
|
"niko/r/reports/c3/test|niko|r|main|20"
|
||||||
|
"niko/r/reports/c4/test|niko|r|main|10"
|
||||||
|
)
|
||||||
|
TO_DELETE=(
|
||||||
|
"niko/r/reports/abc/branch-gone"
|
||||||
|
)
|
||||||
|
|
||||||
|
apply_retention "$CONFIG"
|
||||||
|
|
||||||
|
# 1 pre-existing + 1 commit deleted (c1, oldest of 4, keepMin=3)
|
||||||
|
[ "${#TO_DELETE[@]}" -eq 2 ]
|
||||||
|
[[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# branch_exists — mocking REPO_STATUS / REPO_BRANCHES_CACHE
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@test "branch_exists: branch in list → return 0" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test-token"
|
||||||
|
|
||||||
|
REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x'
|
||||||
|
REPO_STATUS["owner/repo"]="ok"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "main"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: branch not in list → return 1" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test-token"
|
||||||
|
|
||||||
|
REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x'
|
||||||
|
REPO_STATUS["owner/repo"]="ok"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "nonexistent"
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: cert error → exit 1 with [FATAL]" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test"
|
||||||
|
|
||||||
|
REPO_STATUS["owner/repo"]="cert_error"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "any-branch"
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"[FATAL]"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: repo deleted → return 1" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test-token"
|
||||||
|
|
||||||
|
REPO_BRANCHES_CACHE["owner/repo"]="__REPO_DELETED__"
|
||||||
|
REPO_STATUS["owner/repo"]="deleted"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "any-branch"
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: network error → return 0 (fail-safe keep)" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test-token"
|
||||||
|
|
||||||
|
REPO_BRANCHES_CACHE["owner/repo"]="__REPO_ERROR__"
|
||||||
|
REPO_STATUS["owner/repo"]="error"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "any-branch"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: empty GITEA_API_URL → return 0 (skip)" {
|
||||||
|
GITEA_API_URL=""
|
||||||
|
GITEA_TOKEN="test-token"
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "any-branch"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "branch_exists: empty GITEA_TOKEN → return 0 (skip)" {
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN=""
|
||||||
|
|
||||||
|
run branch_exists "owner" "repo" "any-branch"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# repo_branches — git ls-remote error detection patterns
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@test "error detection: 'command not found' does NOT trigger repo deleted" {
|
||||||
|
# This must NOT match — "bash: git: command not found" is NOT a repo deletion
|
||||||
|
local msg="bash: git: command not found"
|
||||||
|
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "error detection: 'fatal: repo not found' triggers repo deleted" {
|
||||||
|
# This MUST match — genuine git error for deleted/missing repo
|
||||||
|
local msg="fatal: repository 'https://gitea.app/owner/repo.git' not found"
|
||||||
|
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "error detection: 'could not read from remote' triggers repo deleted" {
|
||||||
|
local msg="fatal: could not read from remote repository"
|
||||||
|
run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# git ls-remote integration (real git, temp repo)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@test "git ls-remote parsing: lists branches correctly" {
|
||||||
|
local tmpdir=$(mktemp -d)
|
||||||
|
|
||||||
|
git -C "$tmpdir" init -b main source >/dev/null 2>&1
|
||||||
|
git -C "$tmpdir/source" config user.email "test@test"
|
||||||
|
git -C "$tmpdir/source" config user.name "test"
|
||||||
|
git -C "$tmpdir/source" commit --allow-empty -m "init" >/dev/null 2>&1
|
||||||
|
git -C "$tmpdir/source" branch feature/x >/dev/null 2>&1
|
||||||
|
git clone --bare "$tmpdir/source" "$tmpdir/repo.git" >/dev/null 2>&1
|
||||||
|
|
||||||
|
local url="file://$tmpdir/repo.git"
|
||||||
|
local output
|
||||||
|
output=$(git ls-remote --heads "$url" 2>&1)
|
||||||
|
local branches
|
||||||
|
branches=$(echo "$output" | sed -n 's|.*refs/heads/||p')
|
||||||
|
|
||||||
|
echo "$branches" | grep -qxF "main"
|
||||||
|
[ "$?" -eq 0 ]
|
||||||
|
|
||||||
|
echo "$branches" | grep -qxF "feature/x"
|
||||||
|
[ "$?" -eq 0 ]
|
||||||
|
|
||||||
|
! echo "$branches" | grep -qxF "nonexistent"
|
||||||
|
|
||||||
|
rm -rf "$tmpdir"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# repo_branches — retry + error output (using git mock)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@test "repo_branches: success returns branches immediately" {
|
||||||
|
local mockdir=$(mktemp -d)
|
||||||
|
cat > "$mockdir/git" << 'SCRIPT'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
echo "abc123 refs/heads/main"
|
||||||
|
echo "def456 refs/heads/feature/x"
|
||||||
|
SCRIPT
|
||||||
|
chmod +x "$mockdir/git"
|
||||||
|
local save_PATH="$PATH"
|
||||||
|
export PATH="$mockdir:$PATH"
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test"
|
||||||
|
REPO_BRANCHES_CACHE=()
|
||||||
|
REPO_STATUS=()
|
||||||
|
|
||||||
|
run repo_branches "owner" "repo"
|
||||||
|
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"main"* ]]
|
||||||
|
[[ "$output" == *"feature/x"* ]]
|
||||||
|
|
||||||
|
export PATH="$save_PATH"
|
||||||
|
rm -rf "$mockdir"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "repo_branches: retries 3 times on transient error" {
|
||||||
|
local mockdir=$(mktemp -d)
|
||||||
|
cat > "$mockdir/git" << 'SCRIPT'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
echo "call" >> "$MOCKDIR/count"
|
||||||
|
echo "fatal: unable to access 'https://...'" >&2
|
||||||
|
exit 1
|
||||||
|
SCRIPT
|
||||||
|
chmod +x "$mockdir/git"
|
||||||
|
# Inject mockdir path into mock script via env var
|
||||||
|
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||||
|
local save_PATH="$PATH"
|
||||||
|
export PATH="$mockdir:$PATH"
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test"
|
||||||
|
REPO_BRANCHES_CACHE=()
|
||||||
|
REPO_STATUS=()
|
||||||
|
|
||||||
|
local start=$SECONDS
|
||||||
|
run repo_branches "owner" "repo"
|
||||||
|
|
||||||
|
[ "$status" -eq 3 ]
|
||||||
|
[[ "$output" == *"[WARN] git-pages.retention"* ]]
|
||||||
|
[[ "$output" == *"keeping all reports"* ]]
|
||||||
|
[[ "$output" == *"git output:"* ]]
|
||||||
|
[[ "$output" == *"unable to access"* ]]
|
||||||
|
[ $(cat "$mockdir/count" | wc -l) -eq 3 ]
|
||||||
|
[ $(( SECONDS - start )) -ge 18 ]
|
||||||
|
|
||||||
|
export PATH="$save_PATH"
|
||||||
|
rm -rf "$mockdir"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "repo_branches: certificate error → [ERROR] + return 1" {
|
||||||
|
local mockdir=$(mktemp -d)
|
||||||
|
cat > "$mockdir/git" << 'SCRIPT'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
echo "call" >> "$MOCKDIR/count"
|
||||||
|
echo "fatal: unable to access 'https://gitea.app/owner/repo.git/': server certificate verification failed. CAfile: none CRLfile: none" >&2
|
||||||
|
exit 1
|
||||||
|
SCRIPT
|
||||||
|
chmod +x "$mockdir/git"
|
||||||
|
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||||
|
local save_PATH="$PATH"
|
||||||
|
export PATH="$mockdir:$PATH"
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test"
|
||||||
|
REPO_BRANCHES_CACHE=()
|
||||||
|
REPO_STATUS=()
|
||||||
|
|
||||||
|
run repo_branches "owner" "repo"
|
||||||
|
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"[ERROR]"* ]]
|
||||||
|
[[ "$output" == *"certificate verification"* ]]
|
||||||
|
[[ "$output" == *"git output:"* ]]
|
||||||
|
[[ "$output" == *"unable to access"* ]]
|
||||||
|
|
||||||
|
export PATH="$save_PATH"
|
||||||
|
rm -rf "$mockdir"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "repo_branches: repo not found returns immediately (no retry)" {
|
||||||
|
local mockdir=$(mktemp -d)
|
||||||
|
cat > "$mockdir/git" << 'SCRIPT'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
echo "call" >> "$MOCKDIR/count"
|
||||||
|
echo "fatal: repository 'https://gitea.app/owner/repo.git' not found" >&2
|
||||||
|
exit 1
|
||||||
|
SCRIPT
|
||||||
|
chmod +x "$mockdir/git"
|
||||||
|
sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git"
|
||||||
|
local save_PATH="$PATH"
|
||||||
|
export PATH="$mockdir:$PATH"
|
||||||
|
GITEA_API_URL="https://gitea.example.com"
|
||||||
|
GITEA_TOKEN="test"
|
||||||
|
REPO_BRANCHES_CACHE=()
|
||||||
|
REPO_STATUS=()
|
||||||
|
|
||||||
|
run repo_branches "owner" "repo"
|
||||||
|
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"REPO DELETED"* ]]
|
||||||
|
[[ "$output" != *"[WARN]"* ]]
|
||||||
|
[ $(cat "$mockdir/count" | wc -l) -eq 1 ]
|
||||||
|
|
||||||
|
export PATH="$save_PATH"
|
||||||
|
rm -rf "$mockdir"
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user