retention fix

git-pages/files/retention-cleanup.sh

@@ -13,45 +13,8 @@ curl_with_host() {
 
 [ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }
 
-declare -A BRANCH_CACHE
-branch_exists() {
-  local owner="$1" repo="$2" branch="$3" key="${owner}/${repo}/${branch}"
-  local status attempt
-  
-  [ -z "$GITEA_API_URL" ] && return 0
-  [ -z "$GITEA_TOKEN" ] && return 0
-  
-  if [ "${BRANCH_CACHE[$key]:-}" = "1" ]; then
-    return 0
-  fi
-  
-  # Retry up to 2 times on API errors (hardcoded)
-  for attempt in 1 2 3; do
-    status=$(curl -sS -o /dev/null -w "%{http_code}" \
-      -H "Authorization: token ${GITEA_TOKEN}" \
-      "${GITEA_API_URL}/api/v1/repos/${owner}/${repo}/branches/${branch}" 2>/dev/null || echo "000")
-    
-    if [ "$status" = "200" ]; then
-      BRANCH_CACHE[$key]=1
-      return 0
-    fi
-    
-    if [ "$status" = "404" ]; then
-      return 1
-    fi
-    
-    # API error - retry if not last attempt
-    if [ "$attempt" -lt 3 ]; then
-      sleep 10
-      continue
-    fi
-  done
-  
-  # All retries failed - keep report (fail-safe)
-  echo "  WARN: Gitea API error for ${owner}/${repo}/${branch} (status ${status}) after 3 attempts - KEEPING report"
-  BRANCH_CACHE[$key]=1
-  return 0
-}
+declare -A REPO_BRANCHES_CACHE
+declare -A REPO_STATUS
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "$SCRIPT_DIR/retention-lib.sh"
@@ -70,6 +33,7 @@ fi
 echo ""
 echo "=== Phase 1: collect reports ==="
 declare -A SEEN_REPORTS
+declare -A SEEN_ECHO_COMMITS
 declare -a REPORTS
 while IFS= read -r meta_path; do
   report_dir=$(dirname "$meta_path")
@@ -90,34 +54,105 @@ while IFS= read -r meta_path; do
 
   days=$(age_days "$published")
   REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")
-  echo "  ${OWNER}/${REPO} branch=${branch} age=${days}d"
+
+  commit_dir=$(dirname "$report_dir")
+  if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then
+    SEEN_ECHO_COMMITS[$commit_dir]=1
+    echo "  ${commit_dir} branch=${branch} age=${days}d"
+  fi
 done <<< "$META_PATHS"
 
 [ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }
 
 echo ""
-echo "=== Phase 2: check branches in Gitea ==="
+echo "=== Phase 2: check branches/repos in Gitea ==="
+if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then
+  echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2
+  exit 1
+fi
 declare -a TO_DELETE
 declare -a KEEP
+declare -A SEEN_ECHO_BRANCHES
+declare -A SEEN_ECHO_REPO_DELETED
+declare -A UNIQUE_BRANCHES
+declare -A REASON_MAP
+declare -A COMMIT_BRANCH_MAP
+
+# Build commit→branch mapping
+for entry in "${REPORTS[@]}"; do
+  IFS='|' read -r dir _ _ branch _ <<< "$entry"
+  commit_dir=$(dirname "$dir")
+  [ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch
+done
+for entry in "${REPORTS[@]}"; do
+  IFS='|' read -r _ owner repo branch _ <<< "$entry"
+  UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1
+done
+TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]}
+BRANCHES_EXISTING=0
+BRANCH_DELETED_COUNT=0
+REPO_DELETED_COUNT=0
+MAXAGE_DELETED=0
+KEEPMIN_DELETED=0
 for entry in "${REPORTS[@]}"; do
   IFS='|' read -r dir owner repo branch days <<< "$entry"
 
-  if [ -n "$GITEA_API_URL" ] && [ -n "$GITEA_TOKEN" ]; then
-    if branch_exists "$owner" "$repo" "$branch"; then
-      echo "  BRANCH EXISTS: ${owner}/${repo}/${branch}"
-      KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
-    else
-      echo "  BRANCH DELETED: ${owner}/${repo}/${branch} -> DELETE"
-      TO_DELETE+=("$dir")
+  branch_key="${owner}/${repo}/${branch}"
+  if branch_exists "$owner" "$repo" "$branch"; then
+    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
+      SEEN_ECHO_BRANCHES[$branch_key]=1
+      BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1))
+      echo "  BRANCH EXISTS: ${branch_key}"
     fi
-  else
     KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
+  else
+    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
+      SEEN_ECHO_BRANCHES[$branch_key]=1
+      repo_key="${owner}/${repo}"
+      if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then
+        REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1))
+        if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then
+          SEEN_ECHO_REPO_DELETED[$repo_key]=1
+          echo "  REPO DELETED: ${repo_key} -> DELETE ALL"
+        fi
+        reason="repo deleted"
+      else
+        BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1))
+        echo "  BRANCH DELETED: ${branch_key} -> DELETE"
+        reason="branch deleted"
+      fi
+    fi
+    REASON_MAP["$dir"]="$reason"
+    TO_DELETE+=("$dir")
   fi
 done
 
 echo ""
 echo "=== Phase 3: apply retention rules to remaining reports ==="
+PHASE2_DELETED=${#TO_DELETE[@]}
 apply_retention "$CONFIG"
+PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED ))
+
+fmt_num() {
+  local n="$1" out=""
+  [ -z "$n" ] && { echo "?"; return; }
+  n="${n##0}"  # strip leading zeros
+  while [ "${#n}" -gt 3 ]; do
+    out=" ${n: -3}$out"
+    n="${n:0:${#n}-3}"
+  done
+  echo "${n}${out}"
+}
+
+echo ""
+echo "=== Summary ==="
+echo "  Branches:"
+echo "    existing:          $(fmt_num $BRANCHES_EXISTING)"
+echo "    deleted:           $(fmt_num $BRANCH_DELETED_COUNT)"
+echo "    repo gone:         $(fmt_num $REPO_DELETED_COUNT)"
+echo "  Commits:"
+echo "    deleted by maxAge: $(fmt_num $MAXAGE_DELETED)"
+echo "    deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)"
 
 if [ "${#TO_DELETE[@]}" -eq 0 ]; then
   echo "Nothing to delete"
@@ -142,14 +177,37 @@ echo "Downloading archive.tar..."
 HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")
 
 if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
-  echo "Extracting archive..."
+  OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}')
+  echo "Extracting archive (${OLD_KB}kB)..."
   tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"
 
-  for dir in "${TO_DELETE[@]}"; do
-    if [ -d "$SITE_DIR/$dir" ]; then
-      echo "  Removing: $dir"
-      rm -rf "$SITE_DIR/$dir"
+  declare -A GROUP_SEEN
+  declare -A GROUP_LINES
+  for del in "${TO_DELETE[@]}"; do
+    if [ ! -d "$SITE_DIR/$del" ]; then
+      continue
     fi
+
+    commit_dir=$(dirname "$del")
+    branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}"
+    reason="${REASON_MAP[$del]:-?}"
+    repo_path="${del%%/reports/*}"
+    commit_hash="${commit_dir##*/}"
+    key="${repo_path}/${branch} | Reason: ${reason}"
+    seen_key="${key}|${commit_hash}"
+
+    if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then
+      GROUP_SEEN[$seen_key]=1
+      GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash"
+    fi
+
+    rm -rf "$SITE_DIR/$del"
+  done
+  for key in "${!GROUP_LINES[@]}"; do
+    echo "  Removing: ${key}"
+    for hash in ${GROUP_LINES[$key]}; do
+      echo "    commit: ${hash}"
+    done
   done
 else
   echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"
@@ -197,6 +255,7 @@ if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then
 fi
 
 tar -cf "$NEW_TAR" -C "$SITE_DIR" .
+NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}')
 
 echo "PUT: replacing site contents..."
 HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
@@ -208,6 +267,9 @@ HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
 echo "HTTP ${HTTP_CODE}"
 if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
   echo "Site rebuild completed."
+  if [ -n "${OLD_KB:-}" ]; then
+    echo "  archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB"
+  fi
 else
   echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
   exit 1