gitea-ci-library/git-pages/files/retention-cleanup.sh

#!/usr/bin/env bash
set -eo pipefail

PAGES_URL="${PAGES_URL:-http://localhost:3000}"
PAGES_HOST="${PAGES_HOST:?PAGES_HOST is required}"
CONFIG="${RETENTION_CONFIG:-/etc/retention/retention.json}"
GITEA_API_URL="${GITEA_API_URL:-}"
GITEA_TOKEN="${GITEA_TOKEN:-}"

curl_with_host() {
  curl -sS -H "Host: ${PAGES_HOST}" "$@"
}

[ -f "$CONFIG" ] || { echo "ERROR: config missing: $CONFIG" >&2; exit 1; }

declare -A REPO_BRANCHES_CACHE
declare -A REPO_STATUS

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/retention-lib.sh"

echo "Fetching manifest from ${PAGES_URL}/.git-pages/manifest.json"
MANIFEST=$(curl_with_host "${PAGES_URL}/.git-pages/manifest.json")
echo "Manifest loaded"

META_PATHS=$(echo "$MANIFEST" | jq -r '.contents | to_entries[] | select(.key | test("/reports/")) | select(.key | endswith("/.meta")) | .key' 2>/dev/null || true)

if [ -z "$META_PATHS" ]; then
  echo "No .meta files found under /reports/ — nothing to clean"
  exit 0
fi

echo ""
echo "=== Phase 1: collect reports ==="
declare -A SEEN_REPORTS
declare -A SEEN_ECHO_COMMITS
declare -a REPORTS
while IFS= read -r meta_path; do
  report_dir=$(dirname "$meta_path")
  
  # Skip duplicates - same report dir already processed
  [ -z "${SEEN_REPORTS[$report_dir]:-}" ] || continue
  SEEN_REPORTS[$report_dir]=1
  
  parse_path "$report_dir"
  meta_content=$(curl_with_host "${PAGES_URL}/${meta_path}" 2>/dev/null || true)
  [ -n "$meta_content" ] || { echo "  WARN: could not fetch $meta_path"; continue; }

  branch=$(echo "$meta_content" | jq -r '.branch // empty' 2>/dev/null || true)
  published=$(echo "$meta_content" | jq -r '.published_at // empty' 2>/dev/null || true)

  [ -n "$branch" ] || { echo "  WARN: no branch in $meta_path"; continue; }
  [ -n "$published" ] || { echo "  WARN: no published_at in $meta_path"; continue; }

  days=$(age_days "$published")
  REPORTS+=("${report_dir}|${OWNER}|${REPO}|${branch}|${days}")

  commit_dir=$(dirname "$report_dir")
  if [ -z "${SEEN_ECHO_COMMITS[$commit_dir]:-}" ]; then
    SEEN_ECHO_COMMITS[$commit_dir]=1
    echo "  ${commit_dir} branch=${branch} age=${days}d"
  fi
done <<< "$META_PATHS"

[ "${#REPORTS[@]}" -eq 0 ] && { echo "No actionable reports"; exit 0; }

echo ""
echo "=== Phase 2: check branches/repos in Gitea ==="
if [ -z "$GITEA_API_URL" ] || [ -z "$GITEA_TOKEN" ]; then
  echo "ERROR: GITEA_API_URL and GITEA_TOKEN must be set" >&2
  exit 1
fi
declare -a TO_DELETE
declare -a KEEP
declare -A SEEN_ECHO_BRANCHES
declare -A SEEN_ECHO_REPO_DELETED
declare -A UNIQUE_BRANCHES
declare -A REASON_MAP
declare -A COMMIT_BRANCH_MAP

# Build commit→branch mapping
for entry in "${REPORTS[@]}"; do
  IFS='|' read -r dir _ _ branch _ <<< "$entry"
  commit_dir=$(dirname "$dir")
  [ -n "${COMMIT_BRANCH_MAP[$commit_dir]:-}" ] || COMMIT_BRANCH_MAP["$commit_dir"]=$branch
done
for entry in "${REPORTS[@]}"; do
  IFS='|' read -r _ owner repo branch _ <<< "$entry"
  UNIQUE_BRANCHES["${owner}/${repo}/${branch}"]=1
done
TOTAL_BRANCHES=${#UNIQUE_BRANCHES[@]}
BRANCHES_EXISTING=0
BRANCH_DELETED_COUNT=0
REPO_DELETED_COUNT=0
MAXAGE_DELETED=0
KEEPMIN_DELETED=0
for entry in "${REPORTS[@]}"; do
  IFS='|' read -r dir owner repo branch days <<< "$entry"

  branch_key="${owner}/${repo}/${branch}"
  if branch_exists "$owner" "$repo" "$branch"; then
    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
      SEEN_ECHO_BRANCHES[$branch_key]=1
      BRANCHES_EXISTING=$((BRANCHES_EXISTING + 1))
      echo "  BRANCH EXISTS: ${branch_key}"
    fi
    KEEP+=("${dir}|${owner}|${repo}|${branch}|${days}")
  else
    if [ -z "${SEEN_ECHO_BRANCHES[$branch_key]:-}" ]; then
      SEEN_ECHO_BRANCHES[$branch_key]=1
      repo_key="${owner}/${repo}"
      if [ "${REPO_STATUS[$repo_key]:-}" = "deleted" ]; then
        REPO_DELETED_COUNT=$((REPO_DELETED_COUNT + 1))
        if [ -z "${SEEN_ECHO_REPO_DELETED[$repo_key]:-}" ]; then
          SEEN_ECHO_REPO_DELETED[$repo_key]=1
          echo "  REPO DELETED: ${repo_key} -> DELETE ALL"
        fi
        reason="repo deleted"
      else
        BRANCH_DELETED_COUNT=$((BRANCH_DELETED_COUNT + 1))
        echo "  BRANCH DELETED: ${branch_key} -> DELETE"
        reason="branch deleted"
      fi
    fi
    REASON_MAP["$dir"]="$reason"
    TO_DELETE+=("$dir")
  fi
done

echo ""
echo "=== Phase 3: apply retention rules to remaining reports ==="
PHASE2_DELETED=${#TO_DELETE[@]}
apply_retention "$CONFIG"
PHASE3_DELETED=$(( ${#TO_DELETE[@]} - PHASE2_DELETED ))

fmt_num() {
  local n="$1" out=""
  [ -z "$n" ] && { echo "?"; return; }
  n="${n##0}"  # strip leading zeros
  while [ "${#n}" -gt 3 ]; do
    out=" ${n: -3}$out"
    n="${n:0:${#n}-3}"
  done
  echo "${n}${out}"
}

echo ""
echo "=== Summary ==="
echo "  Branches:"
echo "    existing:          $(fmt_num $BRANCHES_EXISTING)"
echo "    deleted:           $(fmt_num $BRANCH_DELETED_COUNT)"
echo "    repo gone:         $(fmt_num $REPO_DELETED_COUNT)"
echo "  Commits:"
echo "    deleted by maxAge: $(fmt_num $MAXAGE_DELETED)"
echo "    deleted by keepMin:$(fmt_num $KEEPMIN_DELETED)"

if [ "${#TO_DELETE[@]}" -eq 0 ]; then
  echo "Nothing to delete"
  exit 0
fi

echo ""
echo "=== Phase 4: full site rebuild ==="
echo "Rebuilding site (${#TO_DELETE[@]} report(s) to delete)..."

ARCHIVE_FILE=$(mktemp)
SITE_DIR=$(mktemp -d)
NEW_TAR=$(mktemp)
cleanup_phase4() {
  rm -f "$ARCHIVE_FILE" "$NEW_TAR"
  rm -rf "$SITE_DIR"
}
trap cleanup_phase4 EXIT

# Try archive.tar first
echo "Downloading archive.tar..."
HTTP_CODE=$(curl_with_host -o "$ARCHIVE_FILE" -w "%{http_code}" -sS "${PAGES_URL}/.git-pages/archive.tar")

if [ "$HTTP_CODE" = "200" ] && tar -tf "$ARCHIVE_FILE" >/dev/null 2>&1; then
  OLD_KB=$(du -sk "$ARCHIVE_FILE" 2>/dev/null | awk '{print $1}')
  echo "Extracting archive (${OLD_KB}kB)..."
  tar -xf "$ARCHIVE_FILE" -C "$SITE_DIR"

  declare -A GROUP_SEEN
  declare -A GROUP_LINES
  for del in "${TO_DELETE[@]}"; do
    if [ ! -d "$SITE_DIR/$del" ]; then
      continue
    fi

    commit_dir=$(dirname "$del")
    branch="${COMMIT_BRANCH_MAP[$commit_dir]:-?}"
    reason="${REASON_MAP[$del]:-?}"
    repo_path="${del%%/reports/*}"
    commit_hash="${commit_dir##*/}"
    key="${repo_path}/${branch} | Reason: ${reason}"
    seen_key="${key}|${commit_hash}"

    if [ -z "${GROUP_SEEN[$seen_key]:-}" ]; then
      GROUP_SEEN[$seen_key]=1
      GROUP_LINES["$key"]="${GROUP_LINES[$key]:-} $commit_hash"
    fi

    rm -rf "$SITE_DIR/$del"
  done
  for key in "${!GROUP_LINES[@]}"; do
    echo "  Removing: ${key}"
    for hash in ${GROUP_LINES[$key]}; do
      echo "    commit: ${hash}"
    done
  done
else
  echo "archive.tar failed (HTTP ${HTTP_CODE}) - falling back to manifest-based rebuild"

  ALL_PATHS=$(echo "$MANIFEST" | jq -r '.contents | keys[]' 2>/dev/null || true)

  if [ -z "$ALL_PATHS" ]; then
    echo "ERROR: no files in manifest - cannot rebuild" >&2
    exit 1
  fi

  EXCLUDE_GREP=""
  for dir in "${TO_DELETE[@]}"; do
    EXCLUDE_GREP="${EXCLUDE_GREP}${EXCLUDE_GREP:+|}^${dir}/"
  done

  if [ -n "$EXCLUDE_GREP" ]; then
    KEEP_PATHS=$(echo "$ALL_PATHS" | grep -v -E "$EXCLUDE_GREP" || true)
  else
    KEEP_PATHS="$ALL_PATHS"
  fi

  if [ -z "$KEEP_PATHS" ]; then
    echo "No files to keep - site will be empty"
    mkdir -p "$SITE_DIR/__placeholder__"
    echo "placeholder" > "$SITE_DIR/__placeholder__/index.html"
  else
    FILE_COUNT=$(echo "$KEEP_PATHS" | wc -l | tr -d ' ')
    echo "Downloading ${FILE_COUNT} file(s)..."
    while IFS= read -r path; do
      [ -z "$path" ] && continue
      dir=$(dirname "$SITE_DIR/$path")
      mkdir -p "$dir"
      curl_with_host -o "$SITE_DIR/$path" -sS "${PAGES_URL}/${path}" || {
        echo "  WARN: failed to download ${path}"
      }
    done <<< "$KEEP_PATHS"
  fi
fi

if [ -z "$(ls -A "$SITE_DIR" 2>/dev/null)" ]; then
  echo "Site is empty - creating placeholder"
  mkdir -p "$SITE_DIR/__placeholder__"
  echo "placeholder" > "$SITE_DIR/__placeholder__/index.html"
fi

tar -cf "$NEW_TAR" -C "$SITE_DIR" .
NEW_KB=$(du -sk "$NEW_TAR" 2>/dev/null | awk '{print $1}')

echo "PUT: replacing site contents..."
HTTP_CODE=$(curl_with_host -X PUT "${PAGES_URL}/" \
  -H "Content-Type: application/x-tar" \
  --data-binary @"${NEW_TAR}" \
  -w "%{http_code}" \
  -o /dev/null)

echo "HTTP ${HTTP_CODE}"
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
  echo "Site rebuild completed."
  if [ -n "${OLD_KB:-}" ]; then
    echo "  archive size: $(fmt_num $OLD_KB)kB → $(fmt_num $NEW_KB)kB"
  fi
else
  echo "ERROR: PUT HTTP ${HTTP_CODE}" >&2
  exit 1
fi