Fix/git pages rentetin create tests (#46)

Co-authored-by: moilanik <niko.moilanen@tietoevry.com> Reviewed-on: #46
2026-06-26 08:07:02 +03:00
parent 1978a995a8
commit 848ba723e4
7 changed files with 974 additions and 108 deletions
@@ -13,75 +13,11 @@ curl_with_host() {

 [ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }

-declare -A BRANCH_CACHE
-branch_exists() {
-  local owner="$1" repo="$2" branch="$3" key="${owner}/${repo}/${branch}"
-  local status attempt
-  
-  [ -z "$GITEA_API_URL" ] && return 0
-  [ -z "$GITEA_TOKEN" ] && return 0
-  
-  if [ "${BRANCH_CACHE[$key]:-}" = "1" ]; then
-    return 0
-  fi
-  
-  # Retry up to 2 times on API errors (hardcoded)
-  for attempt in 1 2 3; do
-    status=$(curl -sS -o /dev/null -w "%{http_code}" \
-      -H "Authorization: token ${GITEA_TOKEN}" \
-      "${GITEA_API_URL}/api/v1/repos/${owner}/${repo}/branches/${branch}" 2>/dev/null || echo "000")
-    
-    if [ "$status" = "200" ]; then
-      BRANCH_CACHE[$key]=1
-      return 0
-    fi
-    
-    if [ "$status" = "404" ]; then
-      return 1
-    fi
-    
-    # API error - retry if not last attempt
-    if [ "$attempt" -lt 3 ]; then
-      sleep 10
-      continue
-    fi
-  done
-  
-  # All retries failed - keep report (fail-safe)
-  echo "  WARN: Gitea API error for ${owner}/${repo}/${branch} (status ${status}) after 3 attempts - KEEPING report"
-  BRANCH_CACHE[$key]=1
-  return 0
-}
+declare -A REPO_BRANCHES_CACHE
+declare -A REPO_STATUS

-default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$CONFIG")
-default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$CONFIG")
-
-rule_max_age() {
-  local branch="$1" v
-  v=$(jq -r --arg b "$branch" '.branches[$b].maxAgeDays // empty' "$CONFIG")
-  [ -n "$v" ] && echo "$v" || echo "$default_max_age"
-}
-
-rule_keep_min() {
-  local branch="$1" v
-  v=$(jq -r --arg b "$branch" '.branches[$b].keepMin // empty' "$CONFIG")
-  [ -n "$v" ] && echo "$v" || echo "$default_keep_min"
-}
-
-age_days() {
-  local published="$1" epoch_pub now
-  epoch_pub=$(date -u -d "$published" +%s 2>/dev/null || echo 0)
-  [ "$epoch_pub" -eq 0 ] && echo 99999 && return
-  now=$(date -u +%s)
-  echo $(( (now - epoch_pub) / 86400 ))
-}
-
-parse_path() {
-  local rel="$1"
-  OWNER="${rel%%/*}"
-  rest="${rel#*/}"
-  REPO="${rest%%/*}"
-}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/retention-lib.sh"

 echo "Fetching manifest from ${PAGES_URL}/.git-pages/manifest.json"
 MANIFEST=$(curl_with_host "${PAGES_URL}/.git-pages/manifest.json")
@@ -97,6 +33,7 @@ fi
 echo ""
 echo "=== Phase 1: collect reports ==="
 declare -A SEEN_REPORTS
+declare -A SEEN_ECHO_COMMITS
 declare -a REPORTS
 while IFS= read -r meta_path; do
  report_dir=$(dirname "$meta_path")
@@ -117,58 +54,105 @@ while IFS= read -r meta_path; do

  days=$(age_days "$published")
  REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")
-  echo "  ${OWNER}/${REPO} branch=${branch} age=${days}d"
+
+  commit_dir=$(dirname "$report_dir")
+  if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then
+    SEEN_ECHO_COMMITS[$commit_dir]=1
+    echo "  ${commit_dir} branch=${branch} age=${days}d"
+  fi
 done <<< "$META_PATHS"

 [ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }

 echo ""
-echo "=== Phase 2: check branches in Gitea ==="
+echo "=== Phase 2: check branches/repos in Gitea ==="
+if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then
+  echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2
+  exit 1
+fi
 declare -a TO_DELETE
 declare -a KEEP
+declare -A SEEN_ECHO_BRANCHES
+declare -A SEEN_ECHO_REPO_DELETED
+declare -A UNIQUE_BRANCHES
+declare -A REASON_MAP
+declare -A COMMIT_BRANCH_MAP
+
+# Build commit→branch mapping
+for entry in "${REPORTS[@]}"; do
+  IFS='|' read -r dir _ _ branch _ <<< "$entry"
+  commit_dir=$(dirname "$dir")
+  [ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch
+done
+for entry in "${REPORTS[@]}"; do
+  IFS='|' read -r _ owner repo branch _ <<< "$entry"
+  UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1
+done
+TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]}
+BRANCHES_EXISTING=0
+BRANCH_DELETED_COUNT=0
+REPO_DELETED_COUNT=0
+MAXAGE_DELETED=0
+KEEPMIN_DELETED=0
 for entry in "${REPORTS[@]}"; do
  IFS='|' read -r dir owner repo branch days <<< "$entry"

-  if [ -n "$GITEA_API_URL" ] && [ -n "$GITEA_TOKEN" ]; then
-    if branch_exists "$owner" "$repo" "$branch"; then
-      echo "  BRANCH EXISTS: ${owner}/${repo}/${branch}"
-      KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
-    else
-      echo "  BRANCH DELETED: ${owner}/${repo}/${branch} -> DELETE"
-      TO_DELETE+=("$dir")
+  branch_key="${owner}/${repo}/${branch}"
+  if branch_exists "$owner" "$repo" "$branch"; then
+    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
+      SEEN_ECHO_BRANCHES[$branch_key]=1
+      BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1))
+      echo "  BRANCH EXISTS: ${branch_key}"
    fi
-  else
    KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
+  else
+    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
+      SEEN_ECHO_BRANCHES[$branch_key]=1
+      repo_key="${owner}/${repo}"
+      if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then
+        REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1))
+        if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then
+          SEEN_ECHO_REPO_DELETED[$repo_key]=1
+          echo "  REPO DELETED: ${repo_key} -> DELETE ALL"
+        fi
+        reason="repo deleted"
+      else
+        BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1))
+        echo "  BRANCH DELETED: ${branch_key} -> DELETE"
+        reason="branch deleted"
+      fi
+    fi
+    REASON_MAP["$dir"]="$reason"
+    TO_DELETE+=("$dir")
  fi
 done

 echo ""
 echo "=== Phase 3: apply retention rules to remaining reports ==="
-declare -A BRANCH_COUNTS
-if [ "${#KEEP[@]}" -gt 0 ]; then
-  IFS=$'\n'
-  for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5rn); do
-    IFS='|' read -r dir owner repo branch days <<< "$entry"
-    max_age=$(rule_max_age "$branch")
-    keep_min=$(rule_keep_min "$branch")
+PHASE2_DELETED=${#TO_DELETE[@]}
+apply_retention "$CONFIG"
+PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED ))

-    if [ "$days" -gt "$max_age" ]; then
-      echo "  DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
-      TO_DELETE+=("$dir")
-      continue
-    fi
-
-    key="${branch}"
-    count="${BRANCH_COUNTS[$key]:-0}"
-    count=$((count + 1))
-    BRANCH_COUNTS["$key"]=$count
-    if [ "$count" -gt "$keep_min" ]; then
-      echo "  DELETE: ${dir} (kept ${keep_min}/${count}, exceeds keepMin, branch ${branch})"
-      TO_DELETE+=("$dir")
-    fi
+fmt_num() {
+  local n="$1" out=""
+  [ -z "$n" ] && { echo "?"; return; }
+  n="${n##0}"  # strip leading zeros
+  while [ "${#n}" -gt 3 ]; do
+    out=" ${n: -3}$out"
+    n="${n:0:${#n}-3}"
  done
-  unset IFS
-fi
+  echo "${n}${out}"
+}
+
+echo ""
+echo "=== Summary ==="
+echo "  Branches:"
+echo "    existing:          $(fmt_num $BRANCHES_EXISTING)"
+echo "    deleted:           $(fmt_num $BRANCH_DELETED_COUNT)"
+echo "    repo gone:         $(fmt_num $REPO_DELETED_COUNT)"
+echo "  Commits:"
+echo "    deleted by maxAge: $(fmt_num $MAXAGE_DELETED)"
+echo "    deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)"

 if [ "${#TO_DELETE[@]}" -eq 0 ]; then
  echo "Nothing to delete"
@@ -193,14 +177,37 @@ echo "Downloading archive.tar..."
 HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")

 if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
-  echo "Extracting archive..."
+  OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}')
+  echo "Extracting archive (${OLD_KB}kB)..."
  tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"

-  for dir in "${TO_DELETE[@]}"; do
-    if [ -d "$SITE_DIR/$dir" ]; then
-      echo "  Removing: $dir"
-      rm -rf "$SITE_DIR/$dir"
+  declare -A GROUP_SEEN
+  declare -A GROUP_LINES
+  for del in "${TO_DELETE[@]}"; do
+    if [ ! -d "$SITE_DIR/$del" ]; then
+      continue
    fi
+
+    commit_dir=$(dirname "$del")
+    branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}"
+    reason="${REASON_MAP[$del]:-?}"
+    repo_path="${del%%/reports/*}"
+    commit_hash="${commit_dir##*/}"
+    key="${repo_path}/${branch} | Reason: ${reason}"
+    seen_key="${key}|${commit_hash}"
+
+    if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then
+      GROUP_SEEN[$seen_key]=1
+      GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash"
+    fi
+
+    rm -rf "$SITE_DIR/$del"
+  done
+  for key in "${!GROUP_LINES[@]}"; do
+    echo "  Removing: ${key}"
+    for hash in ${GROUP_LINES[$key]}; do
+      echo "    commit: ${hash}"
+    done
  done
 else
  echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"
@@ -248,6 +255,7 @@ if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then
 fi

 tar -cf "$NEW_TAR" -C "$SITE_DIR" .
+NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}')

 echo "PUT: replacing site contents..."
 HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
@@ -259,6 +267,9 @@ HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
 echo "HTTP ${HTTP_CODE}"
 if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
  echo "Site rebuild completed."
+  if [ -n "${OLD_KB:-}" ]; then
+    echo "  archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB"
+  fi
 else
  echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
  exit 1
@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# Shared functions for retention-cleanup.sh
+# Can be sourced by tests for unit testing
+
+age_days() {
+  local published="$1" epoch_pub now
+  epoch_pub=$(date -d "$published" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$published" +%s 2>/dev/null || echo 0)
+  [ "$epoch_pub" -eq 0 ] && echo 99999 && return
+  now=$(date -u +%s)
+  echo $(( (now - epoch_pub) / 86400 ))
+}
+
+parse_path() {
+  local rel="$1"
+  OWNER="${rel%%/*}"
+  rest="${rel#*/}"
+  REPO="${rest%%/*}"
+}
+
+read_rule() {
+  local config="$1" branch="$2" key="$3" default="$4"
+  v=$(jq -r --arg b "$branch" --arg k "$key" '.branches[$b][$k] // empty' "$config")
+  [ -n "$v" ] && echo "$v" || echo "$default"
+}
+
+# ---------------------------------------------------------------------------
+# Gitea branch/repo checking via git ls-remote
+# Uses global: GITEA_API_URL, GITEA_TOKEN
+# Sets global: REPO_BRANCHES_CACHE, REPO_STATUS
+# ---------------------------------------------------------------------------
+
+# Fetch all branches for a repo (one git ls-remote call per repo).
+# Sets REPO_STATUS[owner/repo].
+# Echos branch list on success.
+# Returns: 0=ok, 1=deleted, 2=cert_error, 3=error (fail-safe keep)
+repo_branches() {
+  local owner="$1" repo="$2" key="${owner}/${repo}"
+  local attempt output
+
+  [ -z "$GITEA_API_URL" ] && return 0
+  [ -z "$GITEA_TOKEN" ] && return 0
+
+  # Check cached status first (avoids re-running git on every report)
+  case "${REPO_STATUS[$key]:-}" in
+    deleted)    return 1 ;;
+    cert_error) return 2 ;;
+    error)      echo "${REPO_BRANCHES_CACHE[$key]:-}"; return 3 ;;
+  esac
+
+  # Cache hit (success with branch list)
+  [ -n "${REPO_BRANCHES_CACHE[$key]:-}" ] && { echo "${REPO_BRANCHES_CACHE[$key]}"; return 0; }
+
+  local git_host
+  git_host=$(echo "$GITEA_API_URL" | sed -E 's|^https?://||' | sed 's|/.*$||')
+  local git_url="https://token:${GITEA_TOKEN}@${git_host}/${owner}/${repo}.git"
+
+  for attempt in 1 2 3; do
+    output=$(git ls-remote --heads "$git_url" 2>&1) && {
+      local branches
+      branches=$(echo "$output" | sed -n 's|.*refs/heads/||p')
+      REPO_BRANCHES_CACHE[$key]="$branches"
+      REPO_STATUS[$key]="ok"
+      echo "$branches"
+      return 0
+    }
+
+    # Repo deleted → no retry
+    if echo "$output" | grep -qiE "fatal:.*(not found|repository.*not|could not read)"; then
+      REPO_BRANCHES_CACHE[$key]="__REPO_DELETED__"
+      REPO_STATUS[$key]="deleted"
+      echo "  REPO DELETED: ${owner}/${repo}" >&2
+      return 1
+    fi
+
+    [ "$attempt" -lt 3 ] && sleep 10
+  done
+
+  # Certificate verification failure → configuration error, stop
+  if echo "$output" | grep -qi "server certificate verification failed"; then
+    REPO_STATUS[$key]="cert_error"
+    echo "[ERROR] git-pages.retention: certificate verification failed for ${owner}/${repo}" >&2
+    echo "[ERROR] git-pages.retention:   check CA certificates or set GIT_SSL_NO_VERIFY=1" >&2
+    echo "[ERROR] git-pages.retention:   git output:" >&2
+    echo "$output" >&2
+    return 2
+  fi
+
+  # Other network errors → fail-safe keep, continue
+  REPO_BRANCHES_CACHE[$key]="__REPO_ERROR__"
+  REPO_STATUS[$key]="error"
+  echo "[WARN] git-pages.retention: cannot reach Gitea for ${owner}/${repo} — keeping all reports" >&2
+  echo "[WARN] git-pages.retention:   git output:" >&2
+  echo "$output" >&2
+  return 3
+}
+
+# Check if a specific branch exists in a repo.
+# Returns 0 (exists), 1 (not found/deleted).
+# Returns 2 (cert error), 3 (network error).
+branch_exists() {
+  local owner="$1" repo="$2" branch="$3"
+  local branches rc
+
+  [ -z "$GITEA_API_URL" ] && return 0
+  [ -z "$GITEA_TOKEN" ] && return 0
+
+  branches=$(repo_branches "$owner" "$repo")
+  rc=$?
+
+  # Return codes from repo_branches propagate through $() subshell:
+  # 0=ok, 1=deleted, 2=cert_error, 3=error
+  case $rc in
+    2) echo "[FATAL] git-pages.retention: cannot reach Gitea (${owner}/${repo}) — check configuration" >&2
+       exit 1 ;;
+    3) return 0 ;;  # network error → fail-safe keep
+    1) return 1 ;;  # repo/branch gone
+  esac
+
+  echo "$branches" | grep -qxF "$branch"
+}
+
+# Phase 3: apply retention rules to KEEP array, populate TO_DELETE
+# Reads from global KEEP array
+# Populates global TO_DELETE array
+# Usage: apply_retention <config_path>
+apply_retention() {
+  local config="$1"
+  local default_max_age default_keep_min
+  local max_age keep_min key count seen_key commit_dir
+  local entry dir owner repo branch days
+
+  default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config")
+  default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config")
+
+  declare -A BRANCH_COUNTS
+  declare -A SEEN_COMMITS
+  declare -A DELETED_COMMITS
+
+  if [ "${#KEEP[@]}" -eq 0 ]; then
+    return
+  fi
+
+  IFS=$'\n'
+  for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5n); do
+    IFS='|' read -r dir owner repo branch days <<< "$entry"
+
+    max_age=$(read_rule "$config" "$branch" "maxAgeDays" "$default_max_age")
+    keep_min=$(read_rule "$config" "$branch" "keepMin" "$default_keep_min")
+
+    # Age check — per-report-type deletion
+    if [ "$days" -gt "$max_age" ]; then
+      echo "  DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
+      TO_DELETE+=("$dir")
+      REASON_MAP["$dir"]="maxAgeDays exceed"
+      MAXAGE_DELETED=$((MAXAGE_DELETED + 1))
+      continue
+    fi
+
+    # keepMin — per-commit counting
+    commit_dir=$(dirname "$dir")
+    key="$branch"
+    seen_key="${key}|${commit_dir}"
+
+    if [ -z "${SEEN_COMMITS[$seen_key]:-}" ]; then
+      SEEN_COMMITS["$seen_key"]=1
+      count="${BRANCH_COUNTS[$key]:-0}"
+      count=$((count + 1))
+      BRANCH_COUNTS["$key"]=$count
+    else
+      count="${BRANCH_COUNTS[$key]:-0}"
+    fi
+
+    if [ "$count" -gt "$keep_min" ]; then
+      if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then
+        DELETED_COMMITS[$commit_dir]=1
+        echo "  DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})"
+        TO_DELETE+=("$commit_dir")
+        REASON_MAP["$commit_dir"]="keepMin exceed"
+        KEEPMIN_DELETED=$((KEEPMIN_DELETED + 1))
+      fi
+    fi
+  done
+  unset IFS
+}