From 848ba723e441858334830a20976c4e34db144ba7 Mon Sep 17 00:00:00 2001 From: niko Date: Fri, 26 Jun 2026 08:07:02 +0300 Subject: [PATCH] Fix/git pages rentetin create tests (#46) Co-authored-by: moilanik Reviewed-on: https://gitea.app.keskikuja.site/niko/gitea-ci-library/pulls/46 --- git-pages/README.md | 39 ++ git-pages/files/retention-cleanup.sh | 223 +++---- git-pages/files/retention-lib.sh | 184 ++++++ git-pages/templates/deployment.yaml | 9 +- git-pages/templates/retention-configmap.yaml | 2 + git-pages/templates/retention-cronjob.yaml | 2 +- git-pages/tests/retention.bats | 623 +++++++++++++++++++ 7 files changed, 974 insertions(+), 108 deletions(-) create mode 100644 git-pages/files/retention-lib.sh create mode 100644 git-pages/tests/retention.bats diff --git a/git-pages/README.md b/git-pages/README.md index 701eef4..131fbcf 100644 --- a/git-pages/README.md +++ b/git-pages/README.md @@ -149,3 +149,42 @@ curl -X PATCH https://ci-reports.helm-dev.keskikuja.site/owner/repo/commit/sha8/ - `git-pages-publish-token` = plaintext (luetaan Giteaan viedessä) Tarkemmat secret-ohjeet: [docs/secrets.md](docs/secrets.md). + +--- + +## Testaus + +Retention-logiikalle on unit-testit, jotka testaa funktiot ja Phase 3 -säännöt +erikseen ilman ulkoisia riippuvuuksia. + +```bash +cd git-pages +bats tests/retention.bats +``` + +Testit käyttävät `/files/retention-lib.sh` -jaettua kirjastoa, jota myös +`retention-cleanup.sh` sourceaa. Uutta testiä kirjoittaessa: + +1. Luo config `write_config`-helperilla +2. Täytä `KEEP`-array testidatalla (muoto: `dir|owner|repo|branch|days`) +3. Kutsu `apply_retention "$CONFIG"` +4. Tarkista `TO_DELETE`-array ja `$output` + +**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella). + +--- + +## Retention + +Ylläpitoscripti, joka poistaa vanhat raportit git-pagesista retention‑sääntöjen mukaan. +Ajetaan sidecar‑ tai cronjob‑tilassa Kubernetesissa. + +### Air gap -yhteensopimattomuus + +Retention‑kontti asentaa tarvitsemansa työkalut (`curl`, `jq`) ajon aikana +packagemanagerilla (`apt-get` / `apk`). Tämä **ei toimi air gap -ympäristössä**, +jossa konttirekisteriin tai pakettivarastoihin ei ole verkkoyhteyttä. + +**TODO:** Rakenna custom Docker‑image, jossa deps on valmiina: +`FROM alpine:latest && apk add --no-cache curl jq`. +Pushaa omaan rekisteriin ja päivitä `values.yaml`:n `retention.image`. diff --git a/git-pages/files/retention-cleanup.sh b/git-pages/files/retention-cleanup.sh index 653fd6b..0242070 100644 --- a/git-pages/files/retention-cleanup.sh +++ b/git-pages/files/retention-cleanup.sh @@ -13,75 +13,11 @@ curl_with_host() { [ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; } -declare -A BRANCH_CACHE -branch_exists() { - local owner="$1" repo="$2" branch="$3" key="${owner}/${repo}/${branch}" - local status attempt - - [ -z "$GITEA_API_URL" ] && return 0 - [ -z "$GITEA_TOKEN" ] && return 0 - - if [ "${BRANCH_CACHE[$key]:-}" = "1" ]; then - return 0 - fi - - # Retry up to 2 times on API errors (hardcoded) - for attempt in 1 2 3; do - status=$(curl -sS -o /dev/null -w "%{http_code}" \ - -H "Authorization: token ${GITEA_TOKEN}" \ - "${GITEA_API_URL}/api/v1/repos/${owner}/${repo}/branches/${branch}" 2>/dev/null || echo "000") - - if [ "$status" = "200" ]; then - BRANCH_CACHE[$key]=1 - return 0 - fi - - if [ "$status" = "404" ]; then - return 1 - fi - - # API error - retry if not last attempt - if [ "$attempt" -lt 3 ]; then - sleep 10 - continue - fi - done - - # All retries failed - keep report (fail-safe) - echo " WARN: Gitea API error for ${owner}/${repo}/${branch} (status ${status}) after 3 attempts - KEEPING report" - BRANCH_CACHE[$key]=1 - return 0 -} +declare -A REPO_BRANCHES_CACHE +declare -A REPO_STATUS -default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$CONFIG") -default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$CONFIG") - -rule_max_age() { - local branch="$1" v - v=$(jq -r --arg b "$branch" '.branches[$b].maxAgeDays // empty' "$CONFIG") - [ -n "$v" ] && echo "$v" || echo "$default_max_age" -} - -rule_keep_min() { - local branch="$1" v - v=$(jq -r --arg b "$branch" '.branches[$b].keepMin // empty' "$CONFIG") - [ -n "$v" ] && echo "$v" || echo "$default_keep_min" -} - -age_days() { - local published="$1" epoch_pub now - epoch_pub=$(date -u -d "$published" +%s 2>/dev/null || echo 0) - [ "$epoch_pub" -eq 0 ] && echo 99999 && return - now=$(date -u +%s) - echo $(( (now - epoch_pub) / 86400 )) -} - -parse_path() { - local rel="$1" - OWNER="${rel%%/*}" - rest="${rel#*/}" - REPO="${rest%%/*}" -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/retention-lib.sh" echo "Fetching manifest from ${PAGES_URL}/.git-pages/manifest.json" MANIFEST=$(curl_with_host "${PAGES_URL}/.git-pages/manifest.json") @@ -97,6 +33,7 @@ fi echo "" echo "=== Phase 1: collect reports ===" declare -A SEEN_REPORTS +declare -A SEEN_ECHO_COMMITS declare -a REPORTS while IFS= read -r meta_path; do report_dir=$(dirname "$meta_path") @@ -117,58 +54,105 @@ while IFS= read -r meta_path; do days=$(age_days "$published") REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}") - echo " ${OWNER}/${REPO} branch=${branch} age=${days}d" + + commit_dir=$(dirname "$report_dir") + if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then + SEEN_ECHO_COMMITS[$commit_dir]=1 + echo " ${commit_dir} branch=${branch} age=${days}d" + fi done <<< "$META_PATHS" [ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; } echo "" -echo "=== Phase 2: check branches in Gitea ===" +echo "=== Phase 2: check branches/repos in Gitea ===" +if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then + echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2 + exit 1 +fi declare -a TO_DELETE declare -a KEEP +declare -A SEEN_ECHO_BRANCHES +declare -A SEEN_ECHO_REPO_DELETED +declare -A UNIQUE_BRANCHES +declare -A REASON_MAP +declare -A COMMIT_BRANCH_MAP + +# Build commit→branch mapping +for entry in "${REPORTS[@]}"; do + IFS='|' read -r dir _ _ branch _ <<< "$entry" + commit_dir=$(dirname "$dir") + [ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch +done +for entry in "${REPORTS[@]}"; do + IFS='|' read -r _ owner repo branch _ <<< "$entry" + UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1 +done +TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]} +BRANCHES_EXISTING=0 +BRANCH_DELETED_COUNT=0 +REPO_DELETED_COUNT=0 +MAXAGE_DELETED=0 +KEEPMIN_DELETED=0 for entry in "${REPORTS[@]}"; do IFS='|' read -r dir owner repo branch days <<< "$entry" - if [ -n "$GITEA_API_URL" ] && [ -n "$GITEA_TOKEN" ]; then - if branch_exists "$owner" "$repo" "$branch"; then - echo " BRANCH EXISTS: ${owner}/${repo}/${branch}" - KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}") - else - echo " BRANCH DELETED: ${owner}/${repo}/${branch} -> DELETE" - TO_DELETE+=("$dir") + branch_key="${owner}/${repo}/${branch}" + if branch_exists "$owner" "$repo" "$branch"; then + if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then + SEEN_ECHO_BRANCHES[$branch_key]=1 + BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1)) + echo " BRANCH EXISTS: ${branch_key}" fi - else KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}") + else + if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then + SEEN_ECHO_BRANCHES[$branch_key]=1 + repo_key="${owner}/${repo}" + if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then + REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1)) + if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then + SEEN_ECHO_REPO_DELETED[$repo_key]=1 + echo " REPO DELETED: ${repo_key} -> DELETE ALL" + fi + reason="repo deleted" + else + BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1)) + echo " BRANCH DELETED: ${branch_key} -> DELETE" + reason="branch deleted" + fi + fi + REASON_MAP["$dir"]="$reason" + TO_DELETE+=("$dir") fi done echo "" echo "=== Phase 3: apply retention rules to remaining reports ===" -declare -A BRANCH_COUNTS -if [ "${#KEEP[@]}" -gt 0 ]; then - IFS=$'\n' - for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5rn); do - IFS='|' read -r dir owner repo branch days <<< "$entry" - max_age=$(rule_max_age "$branch") - keep_min=$(rule_keep_min "$branch") +PHASE2_DELETED=${#TO_DELETE[@]} +apply_retention "$CONFIG" +PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED )) - if [ "$days" -gt "$max_age" ]; then - echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})" - TO_DELETE+=("$dir") - continue - fi - - key="${branch}" - count="${BRANCH_COUNTS[$key]:-0}" - count=$((count + 1)) - BRANCH_COUNTS["$key"]=$count - if [ "$count" -gt "$keep_min" ]; then - echo " DELETE: ${dir} (kept ${keep_min}/${count}, exceeds keepMin, branch ${branch})" - TO_DELETE+=("$dir") - fi +fmt_num() { + local n="$1" out="" + [ -z "$n" ] && { echo "?"; return; } + n="${n##0}" # strip leading zeros + while [ "${#n}" -gt 3 ]; do + out=" ${n: -3}$out" + n="${n:0:${#n}-3}" done - unset IFS -fi + echo "${n}${out}" +} + +echo "" +echo "=== Summary ===" +echo " Branches:" +echo " existing: $(fmt_num $BRANCHES_EXISTING)" +echo " deleted: $(fmt_num $BRANCH_DELETED_COUNT)" +echo " repo gone: $(fmt_num $REPO_DELETED_COUNT)" +echo " Commits:" +echo " deleted by maxAge: $(fmt_num $MAXAGE_DELETED)" +echo " deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)" if [ "${#TO_DELETE[@]}" -eq 0 ]; then echo "Nothing to delete" @@ -193,14 +177,37 @@ echo "Downloading archive.tar..." HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar") if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then - echo "Extracting archive..." + OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}') + echo "Extracting archive (${OLD_KB}kB)..." tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR" - for dir in "${TO_DELETE[@]}"; do - if [ -d "$SITE_DIR/$dir" ]; then - echo " Removing: $dir" - rm -rf "$SITE_DIR/$dir" + declare -A GROUP_SEEN + declare -A GROUP_LINES + for del in "${TO_DELETE[@]}"; do + if [ ! -d "$SITE_DIR/$del" ]; then + continue fi + + commit_dir=$(dirname "$del") + branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}" + reason="${REASON_MAP[$del]:-?}" + repo_path="${del%%/reports/*}" + commit_hash="${commit_dir##*/}" + key="${repo_path}/${branch} | Reason: ${reason}" + seen_key="${key}|${commit_hash}" + + if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then + GROUP_SEEN[$seen_key]=1 + GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash" + fi + + rm -rf "$SITE_DIR/$del" + done + for key in "${!GROUP_LINES[@]}"; do + echo " Removing: ${key}" + for hash in ${GROUP_LINES[$key]}; do + echo " commit: ${hash}" + done done else echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild" @@ -248,6 +255,7 @@ if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then fi tar -cf "$NEW_TAR" -C "$SITE_DIR" . +NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}') echo "PUT: replacing site contents..." HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \ @@ -259,6 +267,9 @@ HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \ echo "HTTP ${HTTP_CODE}" if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then echo "Site rebuild completed." + if [ -n "${OLD_KB:-}" ]; then + echo " archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB" + fi else echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2 exit 1 diff --git a/git-pages/files/retention-lib.sh b/git-pages/files/retention-lib.sh new file mode 100644 index 0000000..78b5667 --- /dev/null +++ b/git-pages/files/retention-lib.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Shared functions for retention-cleanup.sh +# Can be sourced by tests for unit testing + +age_days() { + local published="$1" epoch_pub now + epoch_pub=$(date -d "$published" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$published" +%s 2>/dev/null || echo 0) + [ "$epoch_pub" -eq 0 ] && echo 99999 && return + now=$(date -u +%s) + echo $(( (now - epoch_pub) / 86400 )) +} + +parse_path() { + local rel="$1" + OWNER="${rel%%/*}" + rest="${rel#*/}" + REPO="${rest%%/*}" +} + +read_rule() { + local config="$1" branch="$2" key="$3" default="$4" + v=$(jq -r --arg b "$branch" --arg k "$key" '.branches[$b][$k] // empty' "$config") + [ -n "$v" ] && echo "$v" || echo "$default" +} + +# --------------------------------------------------------------------------- +# Gitea branch/repo checking via git ls-remote +# Uses global: GITEA_API_URL, GITEA_TOKEN +# Sets global: REPO_BRANCHES_CACHE, REPO_STATUS +# --------------------------------------------------------------------------- + +# Fetch all branches for a repo (one git ls-remote call per repo). +# Sets REPO_STATUS[owner/repo]. +# Echos branch list on success. +# Returns: 0=ok, 1=deleted, 2=cert_error, 3=error (fail-safe keep) +repo_branches() { + local owner="$1" repo="$2" key="${owner}/${repo}" + local attempt output + + [ -z "$GITEA_API_URL" ] && return 0 + [ -z "$GITEA_TOKEN" ] && return 0 + + # Check cached status first (avoids re-running git on every report) + case "${REPO_STATUS[$key]:-}" in + deleted) return 1 ;; + cert_error) return 2 ;; + error) echo "${REPO_BRANCHES_CACHE[$key]:-}"; return 3 ;; + esac + + # Cache hit (success with branch list) + [ -n "${REPO_BRANCHES_CACHE[$key]:-}" ] && { echo "${REPO_BRANCHES_CACHE[$key]}"; return 0; } + + local git_host + git_host=$(echo "$GITEA_API_URL" | sed -E 's|^https?://||' | sed 's|/.*$||') + local git_url="https://token:${GITEA_TOKEN}@${git_host}/${owner}/${repo}.git" + + for attempt in 1 2 3; do + output=$(git ls-remote --heads "$git_url" 2>&1) && { + local branches + branches=$(echo "$output" | sed -n 's|.*refs/heads/||p') + REPO_BRANCHES_CACHE[$key]="$branches" + REPO_STATUS[$key]="ok" + echo "$branches" + return 0 + } + + # Repo deleted → no retry + if echo "$output" | grep -qiE "fatal:.*(not found|repository.*not|could not read)"; then + REPO_BRANCHES_CACHE[$key]="__REPO_DELETED__" + REPO_STATUS[$key]="deleted" + echo " REPO DELETED: ${owner}/${repo}" >&2 + return 1 + fi + + [ "$attempt" -lt 3 ] && sleep 10 + done + + # Certificate verification failure → configuration error, stop + if echo "$output" | grep -qi "server certificate verification failed"; then + REPO_STATUS[$key]="cert_error" + echo "[ERROR] git-pages.retention: certificate verification failed for ${owner}/${repo}" >&2 + echo "[ERROR] git-pages.retention: check CA certificates or set GIT_SSL_NO_VERIFY=1" >&2 + echo "[ERROR] git-pages.retention: git output:" >&2 + echo "$output" >&2 + return 2 + fi + + # Other network errors → fail-safe keep, continue + REPO_BRANCHES_CACHE[$key]="__REPO_ERROR__" + REPO_STATUS[$key]="error" + echo "[WARN] git-pages.retention: cannot reach Gitea for ${owner}/${repo} — keeping all reports" >&2 + echo "[WARN] git-pages.retention: git output:" >&2 + echo "$output" >&2 + return 3 +} + +# Check if a specific branch exists in a repo. +# Returns 0 (exists), 1 (not found/deleted). +# Returns 2 (cert error), 3 (network error). +branch_exists() { + local owner="$1" repo="$2" branch="$3" + local branches rc + + [ -z "$GITEA_API_URL" ] && return 0 + [ -z "$GITEA_TOKEN" ] && return 0 + + branches=$(repo_branches "$owner" "$repo") + rc=$? + + # Return codes from repo_branches propagate through $() subshell: + # 0=ok, 1=deleted, 2=cert_error, 3=error + case $rc in + 2) echo "[FATAL] git-pages.retention: cannot reach Gitea (${owner}/${repo}) — check configuration" >&2 + exit 1 ;; + 3) return 0 ;; # network error → fail-safe keep + 1) return 1 ;; # repo/branch gone + esac + + echo "$branches" | grep -qxF "$branch" +} + +# Phase 3: apply retention rules to KEEP array, populate TO_DELETE +# Reads from global KEEP array +# Populates global TO_DELETE array +# Usage: apply_retention +apply_retention() { + local config="$1" + local default_max_age default_keep_min + local max_age keep_min key count seen_key commit_dir + local entry dir owner repo branch days + + default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config") + default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config") + + declare -A BRANCH_COUNTS + declare -A SEEN_COMMITS + declare -A DELETED_COMMITS + + if [ "${#KEEP[@]}" -eq 0 ]; then + return + fi + + IFS=$'\n' + for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5n); do + IFS='|' read -r dir owner repo branch days <<< "$entry" + + max_age=$(read_rule "$config" "$branch" "maxAgeDays" "$default_max_age") + keep_min=$(read_rule "$config" "$branch" "keepMin" "$default_keep_min") + + # Age check — per-report-type deletion + if [ "$days" -gt "$max_age" ]; then + echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})" + TO_DELETE+=("$dir") + REASON_MAP["$dir"]="maxAgeDays exceed" + MAXAGE_DELETED=$((MAXAGE_DELETED + 1)) + continue + fi + + # keepMin — per-commit counting + commit_dir=$(dirname "$dir") + key="$branch" + seen_key="${key}|${commit_dir}" + + if [ -z "${SEEN_COMMITS[$seen_key]:-}" ]; then + SEEN_COMMITS["$seen_key"]=1 + count="${BRANCH_COUNTS[$key]:-0}" + count=$((count + 1)) + BRANCH_COUNTS["$key"]=$count + else + count="${BRANCH_COUNTS[$key]:-0}" + fi + + if [ "$count" -gt "$keep_min" ]; then + if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then + DELETED_COMMITS[$commit_dir]=1 + echo " DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})" + TO_DELETE+=("$commit_dir") + REASON_MAP["$commit_dir"]="keepMin exceed" + KEEPMIN_DELETED=$((KEEPMIN_DELETED + 1)) + fi + fi + done + unset IFS +} diff --git a/git-pages/templates/deployment.yaml b/git-pages/templates/deployment.yaml index 2c5e722..47278f2 100644 --- a/git-pages/templates/deployment.yaml +++ b/git-pages/templates/deployment.yaml @@ -70,8 +70,15 @@ spec: set -euo pipefail echo "Retention sidecar: installing deps..." apt-get update -qq - apt-get install -y --no-install-recommends curl jq python3 >/dev/null + apt-get install -y --no-install-recommends curl jq git ca-certificates >/dev/null echo "Retention sidecar: ready" + # Sleep until 01:00 so retention runs at night + now_epoch=$(date +%s) + target_epoch=$(date -d "today 01:00:00" +%s) + [ "$target_epoch" -le "$now_epoch" ] && target_epoch=$((target_epoch + 86400)) + sleep_sec=$((target_epoch - now_epoch)) + echo "Retention sidecar: next run in $((sleep_sec / 3600))h (at 01:00)" + sleep $sleep_sec while true; do /scripts/retention-cleanup.sh echo "Retention sidecar: next run in 24h" diff --git a/git-pages/templates/retention-configmap.yaml b/git-pages/templates/retention-configmap.yaml index 506728e..cc59680 100644 --- a/git-pages/templates/retention-configmap.yaml +++ b/git-pages/templates/retention-configmap.yaml @@ -8,6 +8,8 @@ metadata: data: retention.json: | {{- .Values.retention.rules | toJson | nindent 4 }} + retention-lib.sh: | + {{- .Files.Get "files/retention-lib.sh" | nindent 4 }} retention-cleanup.sh: | {{- .Files.Get "files/retention-cleanup.sh" | nindent 4 }} retention-run.sh: | diff --git a/git-pages/templates/retention-cronjob.yaml b/git-pages/templates/retention-cronjob.yaml index c651b99..ce24de6 100644 --- a/git-pages/templates/retention-cronjob.yaml +++ b/git-pages/templates/retention-cronjob.yaml @@ -33,7 +33,7 @@ spec: - | set -euo pipefail apt-get update -qq - apt-get install -y --no-install-recommends curl jq >/dev/null + apt-get install -y --no-install-recommends curl jq git >/dev/null chmod +x /scripts/retention-run.sh /scripts/retention-cleanup.sh /scripts/retention-run.sh env: diff --git a/git-pages/tests/retention.bats b/git-pages/tests/retention.bats new file mode 100644 index 0000000..490cd9f --- /dev/null +++ b/git-pages/tests/retention.bats @@ -0,0 +1,623 @@ +#!/usr/bin/env bats + +setup() { + source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh" + declare -gA REPO_BRANCHES_CACHE + declare -gA REPO_STATUS + declare -gA REASON_MAP + MAXAGE_DELETED=0 + KEEPMIN_DELETED=0 + CONFIG=$(mktemp) +} + +teardown() { + rm -f "$CONFIG" +} + +write_config() { + cat > "$CONFIG" +} + +# --------------------------------------------------------------------------- +# read_rule +# --------------------------------------------------------------------------- + +@test "read_rule returns default when branch has no override" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + result=$(read_rule "$CONFIG" "nonexistent" "maxAgeDays" 90) + [ "$result" = "90" ] +} + +@test "read_rule returns branch-specific value" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365,"keepMin":20}}} +EOF + result=$(read_rule "$CONFIG" "main" "keepMin" 5) + [ "$result" = "20" ] +} + +@test "read_rule returns default for undefined key even if branch exists" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365}}} +EOF + result=$(read_rule "$CONFIG" "main" "keepMin" 5) + [ "$result" = "5" ] +} + +# --------------------------------------------------------------------------- +# parse_path +# --------------------------------------------------------------------------- + +@test "parse_path extracts owner and repo" { + parse_path "my-owner/my-repo/reports/abc123/go-test-unit" + [ "$OWNER" = "my-owner" ] + [ "$REPO" = "my-repo" ] +} + +@test "parse_path handles owner with hyphens" { + parse_path "niko/agent-platform/reports/abc1234/go-test-bdd" + [ "$OWNER" = "niko" ] + [ "$REPO" = "agent-platform" ] +} + +# --------------------------------------------------------------------------- +# apply_retention — keepMin per commit +# --------------------------------------------------------------------------- + +@test "keepMin: 6 commits × 4 types, keepMin=10 → all kept (6 commits < 10)" { + # With old per-file counting, 24 files > 10 keepMin would delete 14. + # With per-commit counting, 6 commits < 10 keepMin keeps everything. + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}} +EOF + + KEEP=() + local -a commits=(c1 c2 c3 c4 c5 c6) + local -a ages=(100 80 60 40 20 5) + local -a types=(go-test-bdd go-test-unit helm-lint helm-kubeconform) + for i in "${!commits[@]}"; do + for t in "${types[@]}"; do + KEEP+=("niko/agent-platform/reports/${commits[$i]}/$t|niko|agent-platform|main|${ages[$i]}") + done + done + + TO_DELETE=() + apply_retention "$CONFIG" + + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "keepMin: 8 commits × 1 type, keepMin=5 → deletes 3 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":5}}} +EOF + + KEEP=() + local -a ages=(80 70 60 50 40 30 20 10) + for i in "${!ages[@]}"; do + KEEP+=("niko/r/reports/c$((i+1))/test|niko|r|main|${ages[$i]}") + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # 5 newest (c8-c4) kept, 3 oldest (c3,c2,c1) deleted + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]] +} + +@test "keepMin: 12 commits × 1 type, keepMin=5 → keeps 5 newest, deletes 7 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + for i in $(seq 1 12); do + KEEP+=("niko/r/reports/c${i}/test|niko|r|feature/foo|$(( 13 - i ))") + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # 7 oldest commits deleted (12 - 5 = 7) + [ "${#TO_DELETE[@]}" -eq 7 ] + # Oldest 7 should be c1..c7 (highest days = oldest = processed last after sort) + # Sort is ascending by days, so processed as c12(1d), c11(2d), ..., c1(12d) + # keepMin=5: c12-c8 kept, c7-c1 deleted + [[ "${TO_DELETE[0]}" == "niko/r/reports/c7" ]] + [[ "${TO_DELETE[6]}" == "niko/r/reports/c1" ]] +} + +@test "keepMin: 2 commits × 3 types, keepMin=5 → all kept (2 < 5)" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + KEEP+=("niko/r/reports/c1/test-a|niko|r|main|30") + KEEP+=("niko/r/reports/c1/test-b|niko|r|main|30") + KEEP+=("niko/r/reports/c1/test-c|niko|r|main|30") + KEEP+=("niko/r/reports/c2/test-a|niko|r|main|10") + KEEP+=("niko/r/reports/c2/test-b|niko|r|main|10") + KEEP+=("niko/r/reports/c2/test-c|niko|r|main|10") + + TO_DELETE=() + apply_retention "$CONFIG" + + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "keepMin: 6 commits × 2 types, keepMin=3 → keeps 3 newest, deletes 3 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}} +EOF + + KEEP=() + local -a commits=(c1 c2 c3 c4 c5 c6) + local -a ages=(60 50 40 30 20 10) + local -a types=(jest pytest) + for i in "${!commits[@]}"; do + for t in "${types[@]}"; do + KEEP+=("niko/r/reports/${commits[$i]}/$t|niko|r|feature/x|${ages[$i]}") + done + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # Sort by days ascending: c6(10d), c5(20d), c4(30d), c3(40d), c2(50d), c1(60d) + # keepMin=3: c6,c5,c4 kept; c3,c2,c1 deleted + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — maxAge +# --------------------------------------------------------------------------- + +@test "maxAge: report exceeding maxAge is deleted" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|50" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # c1 (100d) > 90, deleted; c2 (50d) < 90, kept + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test" ]] +} + +@test "maxAge deletes report-level dir, not commit-level" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":30,"keepMin":5}}} +EOF + + KEEP=( + "niko/r/reports/c1/test-a|niko|r|main|100" + "niko/r/reports/c1/test-b|niko|r|main|20" + "niko/r/reports/c2/test-a|niko|r|main|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Only test-a for c1 is old; test-b for c1 is young, c2 is young + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test-a" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — maxAge + keepMin interaction +# --------------------------------------------------------------------------- + +@test "maxAge takes precedence over keepMin — aged report deleted, not counted in keepMin" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":30,"keepMin":2}}} +EOF + + # 3 commits, 1 type each. c1 is old (100d), c2 and c3 are young. + # With keepMin=2: c1 should be deleted by maxAge, c2 and c3 kept. + # Without the continue after maxAge check, c1 would consume a keepMin slot. + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|10" + "niko/r/reports/c3/test|niko|r|main|5" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # c1 deleted by maxAge, c2 and c3 within keepMin=2 + [ "${#TO_DELETE[@]}" -eq 1 ] +} + +# --------------------------------------------------------------------------- +# apply_retention — sorting (newest first) +# --------------------------------------------------------------------------- + +@test "sort order: newest commits processed first within same branch" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":2}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|50" + "niko/r/reports/c3/test|niko|r|main|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Sort by days ascending: c3(10d) 1st, c2(50d) 2nd, c1(100d) 3rd + # keepMin=2: c3 and c2 kept, c1 deleted + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]] +} + +@test "sort order: branches sorted alphabetically" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":1}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|z-branch|50" + "niko/r/reports/c2/test|niko|r|a-branch|60" + "niko/r/reports/c3/test|niko|r|m-branch|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Alphabetical: a-branch, m-branch, z-branch + # Each has 1 commit, keepMin=1 → nothing deleted + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "multi-branch: each branch has own keepMin counter" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":2}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|branch-a|30" + "niko/r/reports/c2/test|niko|r|branch-a|20" + "niko/r/reports/c3/test|niko|r|branch-a|10" + "niko/r/reports/c4/test|niko|r|branch-b|60" + "niko/r/reports/c5/test|niko|r|branch-b|50" + "niko/r/reports/c6/test|niko|r|branch-b|40" + "niko/r/reports/c7/test|niko|r|branch-b|30" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # branch-a: 3 reports → keep 2 newest (c2,c3), delete 1 oldest (c1) + # branch-b: 4 reports → keep 2 newest (c6,c7), delete 2 oldest (c4,c5) + # Actually: Sort is by branch, then by days ascending + # branch-a processed first: c3(10d) 1st, c2(20d) 2nd (keep), c1(30d) 3rd (delete) + # branch-b processed next: c7(30d) 1st, c6(40d) 2nd (keep), c5(50d) 3rd (delete), c4(60d) 4th (delete) + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c5" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c4" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — empty / edge cases +# --------------------------------------------------------------------------- + +@test "empty KEEP array → nothing deleted" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + TO_DELETE=() + apply_retention "$CONFIG" + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "TO_DELETE preserves Phase 2 entries after apply_retention (no new deletions)" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|10" + "niko/r/reports/c2/test|niko|r|main|5" + ) + TO_DELETE=( + "niko/r/reports/abc/branch-gone" + "niko/r/reports/def/repo-gone" + ) + + apply_retention "$CONFIG" + + [ "${#TO_DELETE[@]}" -eq 2 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/def/repo-gone" ]] +} + +@test "TO_DELETE preserves Phase 2 entries AND adds retention deletions" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|40" + "niko/r/reports/c2/test|niko|r|main|30" + "niko/r/reports/c3/test|niko|r|main|20" + "niko/r/reports/c4/test|niko|r|main|10" + ) + TO_DELETE=( + "niko/r/reports/abc/branch-gone" + ) + + apply_retention "$CONFIG" + + # 1 pre-existing + 1 commit deleted (c1, oldest of 4, keepMin=3) + [ "${#TO_DELETE[@]}" -eq 2 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/abc/branch-gone" ]] +} + +# --------------------------------------------------------------------------- +# branch_exists — mocking REPO_STATUS / REPO_BRANCHES_CACHE +# --------------------------------------------------------------------------- + +@test "branch_exists: branch in list → return 0" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test-token" + + REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x' + REPO_STATUS["owner/repo"]="ok" + + run branch_exists "owner" "repo" "main" + [ "$status" -eq 0 ] +} + +@test "branch_exists: branch not in list → return 1" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test-token" + + REPO_BRANCHES_CACHE["owner/repo"]=$'main\nfeature/x' + REPO_STATUS["owner/repo"]="ok" + + run branch_exists "owner" "repo" "nonexistent" + [ "$status" -eq 1 ] +} + +@test "branch_exists: cert error → exit 1 with [FATAL]" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test" + + REPO_STATUS["owner/repo"]="cert_error" + + run branch_exists "owner" "repo" "any-branch" + [ "$status" -eq 1 ] + [[ "$output" == *"[FATAL]"* ]] +} + +@test "branch_exists: repo deleted → return 1" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test-token" + + REPO_BRANCHES_CACHE["owner/repo"]="__REPO_DELETED__" + REPO_STATUS["owner/repo"]="deleted" + + run branch_exists "owner" "repo" "any-branch" + [ "$status" -eq 1 ] +} + +@test "branch_exists: network error → return 0 (fail-safe keep)" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test-token" + + REPO_BRANCHES_CACHE["owner/repo"]="__REPO_ERROR__" + REPO_STATUS["owner/repo"]="error" + + run branch_exists "owner" "repo" "any-branch" + [ "$status" -eq 0 ] +} + +@test "branch_exists: empty GITEA_API_URL → return 0 (skip)" { + GITEA_API_URL="" + GITEA_TOKEN="test-token" + + run branch_exists "owner" "repo" "any-branch" + [ "$status" -eq 0 ] +} + +@test "branch_exists: empty GITEA_TOKEN → return 0 (skip)" { + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="" + + run branch_exists "owner" "repo" "any-branch" + [ "$status" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# repo_branches — git ls-remote error detection patterns +# --------------------------------------------------------------------------- + +@test "error detection: 'command not found' does NOT trigger repo deleted" { + # This must NOT match — "bash: git: command not found" is NOT a repo deletion + local msg="bash: git: command not found" + run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg" + [ "$status" -eq 1 ] +} + +@test "error detection: 'fatal: repo not found' triggers repo deleted" { + # This MUST match — genuine git error for deleted/missing repo + local msg="fatal: repository 'https://gitea.app/owner/repo.git' not found" + run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg" + [ "$status" -eq 0 ] +} + +@test "error detection: 'could not read from remote' triggers repo deleted" { + local msg="fatal: could not read from remote repository" + run grep -qiE "fatal:.*(not found|repository.*not|could not read)" <<< "$msg" + [ "$status" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# git ls-remote integration (real git, temp repo) +# --------------------------------------------------------------------------- + +@test "git ls-remote parsing: lists branches correctly" { + local tmpdir=$(mktemp -d) + + git -C "$tmpdir" init -b main source >/dev/null 2>&1 + git -C "$tmpdir/source" config user.email "test@test" + git -C "$tmpdir/source" config user.name "test" + git -C "$tmpdir/source" commit --allow-empty -m "init" >/dev/null 2>&1 + git -C "$tmpdir/source" branch feature/x >/dev/null 2>&1 + git clone --bare "$tmpdir/source" "$tmpdir/repo.git" >/dev/null 2>&1 + + local url="file://$tmpdir/repo.git" + local output + output=$(git ls-remote --heads "$url" 2>&1) + local branches + branches=$(echo "$output" | sed -n 's|.*refs/heads/||p') + + echo "$branches" | grep -qxF "main" + [ "$?" -eq 0 ] + + echo "$branches" | grep -qxF "feature/x" + [ "$?" -eq 0 ] + + ! echo "$branches" | grep -qxF "nonexistent" + + rm -rf "$tmpdir" +} + +# --------------------------------------------------------------------------- +# repo_branches — retry + error output (using git mock) +# --------------------------------------------------------------------------- + +@test "repo_branches: success returns branches immediately" { + local mockdir=$(mktemp -d) + cat > "$mockdir/git" << 'SCRIPT' +#!/usr/bin/env bash +echo "abc123 refs/heads/main" +echo "def456 refs/heads/feature/x" +SCRIPT + chmod +x "$mockdir/git" + local save_PATH="$PATH" + export PATH="$mockdir:$PATH" + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test" + REPO_BRANCHES_CACHE=() + REPO_STATUS=() + + run repo_branches "owner" "repo" + + [ "$status" -eq 0 ] + [[ "$output" == *"main"* ]] + [[ "$output" == *"feature/x"* ]] + + export PATH="$save_PATH" + rm -rf "$mockdir" +} + +@test "repo_branches: retries 3 times on transient error" { + local mockdir=$(mktemp -d) + cat > "$mockdir/git" << 'SCRIPT' +#!/usr/bin/env bash +echo "call" >> "$MOCKDIR/count" +echo "fatal: unable to access 'https://...'" >&2 +exit 1 +SCRIPT + chmod +x "$mockdir/git" + # Inject mockdir path into mock script via env var + sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git" + local save_PATH="$PATH" + export PATH="$mockdir:$PATH" + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test" + REPO_BRANCHES_CACHE=() + REPO_STATUS=() + + local start=$SECONDS + run repo_branches "owner" "repo" + + [ "$status" -eq 3 ] + [[ "$output" == *"[WARN] git-pages.retention"* ]] + [[ "$output" == *"keeping all reports"* ]] + [[ "$output" == *"git output:"* ]] + [[ "$output" == *"unable to access"* ]] + [ $(cat "$mockdir/count" | wc -l) -eq 3 ] + [ $(( SECONDS - start )) -ge 18 ] + + export PATH="$save_PATH" + rm -rf "$mockdir" +} + +@test "repo_branches: certificate error → [ERROR] + return 1" { + local mockdir=$(mktemp -d) + cat > "$mockdir/git" << 'SCRIPT' +#!/usr/bin/env bash +echo "call" >> "$MOCKDIR/count" +echo "fatal: unable to access 'https://gitea.app/owner/repo.git/': server certificate verification failed. CAfile: none CRLfile: none" >&2 +exit 1 +SCRIPT + chmod +x "$mockdir/git" + sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git" + local save_PATH="$PATH" + export PATH="$mockdir:$PATH" + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test" + REPO_BRANCHES_CACHE=() + REPO_STATUS=() + + run repo_branches "owner" "repo" + + [ "$status" -eq 2 ] + [[ "$output" == *"[ERROR]"* ]] + [[ "$output" == *"certificate verification"* ]] + [[ "$output" == *"git output:"* ]] + [[ "$output" == *"unable to access"* ]] + + export PATH="$save_PATH" + rm -rf "$mockdir" +} + +@test "repo_branches: repo not found returns immediately (no retry)" { + local mockdir=$(mktemp -d) + cat > "$mockdir/git" << 'SCRIPT' +#!/usr/bin/env bash +echo "call" >> "$MOCKDIR/count" +echo "fatal: repository 'https://gitea.app/owner/repo.git' not found" >&2 +exit 1 +SCRIPT + chmod +x "$mockdir/git" + sed -i '' "s|\$MOCKDIR|$mockdir|g" "$mockdir/git" + local save_PATH="$PATH" + export PATH="$mockdir:$PATH" + GITEA_API_URL="https://gitea.example.com" + GITEA_TOKEN="test" + REPO_BRANCHES_CACHE=() + REPO_STATUS=() + + run repo_branches "owner" "repo" + + [ "$status" -eq 1 ] + [[ "$output" == *"REPO DELETED"* ]] + [[ "$output" != *"[WARN]"* ]] + [ $(cat "$mockdir/count" | wc -l) -eq 1 ] + + export PATH="$save_PATH" + rm -rf "$mockdir" +}