retention korjaus testien avulla

git-pages/README.md

@@ -149,3 +149,25 @@ curl -X PATCH https://ci-reports.helm-dev.keskikuja.site/owner/repo/commit/sha8/
 - `git-pages-publish-token` = plaintext (luetaan Giteaan viedessä)
 
 Tarkemmat secret-ohjeet: [docs/secrets.md](docs/secrets.md).
+
+---
+
+## Testaus
+
+Retention-logiikalle on unit-testit, jotka testaa funktiot ja Phase 3 -säännöt
+erikseen ilman ulkoisia riippuvuuksia.
+
+```bash
+cd git-pages
+bats tests/retention.bats
+```
+
+Testit käyttävät `<root>/files/retention-lib.sh` -jaettua kirjastoa, jota myös
+`retention-cleanup.sh` sourceaa. Uutta testiä kirjoittaessa:
+
+1. Luo config `write_config`-helperilla
+2. Täytä `KEEP`-array testidatalla (muoto: `dir|owner|repo|branch|days`)
+3. Kutsu `apply_retention "$CONFIG"`
+4. Tarkista `TO_DELETE`-array ja `$output`
+
+**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella).

git-pages/files/retention-cleanup.sh

@@ -53,35 +53,8 @@ branch_exists() {
   return 0
 }
 
-default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$CONFIG")
-default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$CONFIG")
-
-rule_max_age() {
-  local branch="$1" v
-  v=$(jq -r --arg b "$branch" '.branches[$b].maxAgeDays // empty' "$CONFIG")
-  [ -n "$v" ] && echo "$v" || echo "$default_max_age"
-}
-
-rule_keep_min() {
-  local branch="$1" v
-  v=$(jq -r --arg b "$branch" '.branches[$b].keepMin // empty' "$CONFIG")
-  [ -n "$v" ] && echo "$v" || echo "$default_keep_min"
-}
-
-age_days() {
-  local published="$1" epoch_pub now
-  epoch_pub=$(date -u -d "$published" +%s 2>/dev/null || echo 0)
-  [ "$epoch_pub" -eq 0 ] && echo 99999 && return
-  now=$(date -u +%s)
-  echo $(( (now - epoch_pub) / 86400 ))
-}
-
-parse_path() {
-  local rel="$1"
-  OWNER="${rel%%/*}"
-  rest="${rel#*/}"
-  REPO="${rest%%/*}"
-}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/retention-lib.sh"
 
 echo "Fetching manifest from ${PAGES_URL}/.git-pages/manifest.json"
 MANIFEST=$(curl_with_host "${PAGES_URL}/.git-pages/manifest.json")
@@ -144,31 +117,7 @@ done
 
 echo ""
 echo "=== Phase 3: apply retention rules to remaining reports ==="
-declare -A BRANCH_COUNTS
-if [ "${#KEEP[@]}" -gt 0 ]; then
-  IFS=$'\n'
-  for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5rn); do
-    IFS='|' read -r dir owner repo branch days <<< "$entry"
-    max_age=$(rule_max_age "$branch")
-    keep_min=$(rule_keep_min "$branch")
-
-    if [ "$days" -gt "$max_age" ]; then
-      echo "  DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
-      TO_DELETE+=("$dir")
-      continue
-    fi
-
-    key="${branch}"
-    count="${BRANCH_COUNTS[$key]:-0}"
-    count=$((count + 1))
-    BRANCH_COUNTS["$key"]=$count
-    if [ "$count" -gt "$keep_min" ]; then
-      echo "  DELETE: ${dir} (kept ${keep_min}/${count}, exceeds keepMin, branch ${branch})"
-      TO_DELETE+=("$dir")
-    fi
-  done
-  unset IFS
-fi
+apply_retention "$CONFIG"
 
 if [ "${#TO_DELETE[@]}" -eq 0 ]; then
   echo "Nothing to delete"

git-pages/files/retention-lib.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Shared functions for retention-cleanup.sh
+# Can be sourced by tests for unit testing
+
+age_days() {
+  local published="$1" epoch_pub now
+  epoch_pub=$(date -d "$published" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$published" +%s 2>/dev/null || echo 0)
+  [ "$epoch_pub" -eq 0 ] && echo 99999 && return
+  now=$(date -u +%s)
+  echo $(( (now - epoch_pub) / 86400 ))
+}
+
+parse_path() {
+  local rel="$1"
+  OWNER="${rel%%/*}"
+  rest="${rel#*/}"
+  REPO="${rest%%/*}"
+}
+
+read_rule() {
+  local config="$1" branch="$2" key="$3" default="$4"
+  v=$(jq -r --arg b "$branch" --arg k "$key" '.branches[$b][$k] // empty' "$config")
+  [ -n "$v" ] && echo "$v" || echo "$default"
+}
+
+# Phase 3: apply retention rules to KEEP array, populate TO_DELETE
+# Reads from global KEEP array
+# Populates global TO_DELETE array
+# Usage: apply_retention <config_path>
+apply_retention() {
+  local config="$1"
+  local default_max_age default_keep_min
+  local max_age keep_min key count seen_key commit_dir report_type path
+  local entry dir owner repo branch days
+
+  default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config")
+  default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config")
+
+  TO_DELETE=()
+  declare -A BRANCH_COUNTS
+  declare -A SEEN_COMMITS
+  declare -A DELETED_COMMITS
+
+  if [ "${#KEEP[@]}" -eq 0 ]; then
+    return
+  fi
+
+  IFS=$'\n'
+  for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5n); do
+    IFS='|' read -r dir owner repo branch days <<< "$entry"
+
+    max_age=$(read_rule "$config" "$branch" "maxAgeDays" "$default_max_age")
+    keep_min=$(read_rule "$config" "$branch" "keepMin" "$default_keep_min")
+
+    # Age check — per-report-type deletion
+    if [ "$days" -gt "$max_age" ]; then
+      echo "  DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})"
+      TO_DELETE+=("$dir")
+      continue
+    fi
+
+    # keepMin — per-commit counting
+    commit_dir=$(dirname "$dir")
+    key="$branch"
+    seen_key="${key}|${commit_dir}"
+
+    if [ -z "${SEEN_COMMITS[$seen_key]:-}" ]; then
+      SEEN_COMMITS["$seen_key"]=1
+      count="${BRANCH_COUNTS[$key]:-0}"
+      count=$((count + 1))
+      BRANCH_COUNTS["$key"]=$count
+    else
+      count="${BRANCH_COUNTS[$key]:-0}"
+    fi
+
+    if [ "$count" -gt "$keep_min" ]; then
+      if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then
+        DELETED_COMMITS[$commit_dir]=1
+        report_type="${dir##*/}"
+        echo "  DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})"
+        TO_DELETE+=("$commit_dir")
+      fi
+    fi
+  done
+  unset IFS
+}

git-pages/templates/retention-configmap.yaml

@@ -8,6 +8,8 @@ metadata:
 data:
   retention.json: |
     {{- .Values.retention.rules | toJson | nindent 4 }}
+  retention-lib.sh: |
+    {{- .Files.Get "files/retention-lib.sh" | nindent 4 }}
   retention-cleanup.sh: |
     {{- .Files.Get "files/retention-cleanup.sh" | nindent 4 }}
   retention-run.sh: |

git-pages/tests/retention.bats

@@ -0,0 +1,328 @@
+#!/usr/bin/env bats
+
+setup() {
+  # Copy the retention lib to a temp location so it's easier to source
+  # (bats runs each test in its own dir)
+  source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh"
+  CONFIG=$(mktemp)
+}
+
+teardown() {
+  rm -f "$CONFIG"
+}
+
+write_config() {
+  cat > "$CONFIG"
+}
+
+# ---------------------------------------------------------------------------
+# read_rule
+# ---------------------------------------------------------------------------
+
+@test "read_rule returns default when branch has no override" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}}
+EOF
+  result=$(read_rule "$CONFIG" "nonexistent" "maxAgeDays" 90)
+  [ "$result" = "90" ]
+}
+
+@test "read_rule returns branch-specific value" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365,"keepMin":20}}}
+EOF
+  result=$(read_rule "$CONFIG" "main" "keepMin" 5)
+  [ "$result" = "20" ]
+}
+
+@test "read_rule returns default for undefined key even if branch exists" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365}}}
+EOF
+  result=$(read_rule "$CONFIG" "main" "keepMin" 5)
+  [ "$result" = "5" ]
+}
+
+# ---------------------------------------------------------------------------
+# parse_path
+# ---------------------------------------------------------------------------
+
+@test "parse_path extracts owner and repo" {
+  parse_path "my-owner/my-repo/reports/abc123/go-test-unit"
+  [ "$OWNER" = "my-owner" ]
+  [ "$REPO" = "my-repo" ]
+}
+
+@test "parse_path handles owner with hyphens" {
+  parse_path "niko/agent-platform/reports/abc1234/go-test-bdd"
+  [ "$OWNER" = "niko" ]
+  [ "$REPO" = "agent-platform" ]
+}
+
+# ---------------------------------------------------------------------------
+# apply_retention — keepMin per commit
+# ---------------------------------------------------------------------------
+
+@test "keepMin: 6 commits × 4 types, keepMin=10 → all kept (6 commits < 10)" {
+  # With old per-file counting, 24 files > 10 keepMin would delete 14.
+  # With per-commit counting, 6 commits < 10 keepMin keeps everything.
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}}
+EOF
+
+  KEEP=()
+  local -a commits=(c1 c2 c3 c4 c5 c6)
+  local -a ages=(100 80 60 40 20 5)
+  local -a types=(go-test-bdd go-test-unit helm-lint helm-kubeconform)
+  for i in "${!commits[@]}"; do
+    for t in "${types[@]}"; do
+      KEEP+=("niko/agent-platform/reports/${commits[$i]}/$t|niko|agent-platform|main|${ages[$i]}")
+    done
+  done
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  [ "${#TO_DELETE[@]}" -eq 0 ]
+}
+
+@test "keepMin: 8 commits × 1 type, keepMin=5 → deletes 3 oldest" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":365,"keepMin":5}}}
+EOF
+
+  KEEP=()
+  local -a ages=(80 70 60 50 40 30 20 10)
+  for i in "${!ages[@]}"; do
+    KEEP+=("niko/r/reports/c$((i+1))/test|niko|r|main|${ages[$i]}")
+  done
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # 5 newest (c8-c4) kept, 3 oldest (c3,c2,c1) deleted
+  [ "${#TO_DELETE[@]}" -eq 3 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]]
+  [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]]
+  [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]]
+}
+
+@test "keepMin: 12 commits × 1 type, keepMin=5 → keeps 5 newest, deletes 7 oldest" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}}
+EOF
+
+  KEEP=()
+  for i in $(seq 1 12); do
+    KEEP+=("niko/r/reports/c${i}/test|niko|r|feature/foo|$(( 13 - i ))")
+  done
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # 7 oldest commits deleted (12 - 5 = 7)
+  [ "${#TO_DELETE[@]}" -eq 7 ]
+  # Oldest 7 should be c1..c7 (highest days = oldest = processed last after sort)
+  # Sort is ascending by days, so processed as c12(1d), c11(2d), ..., c1(12d)
+  # keepMin=5: c12-c8 kept, c7-c1 deleted
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c7" ]]
+  [[ "${TO_DELETE[6]}" == "niko/r/reports/c1" ]]
+}
+
+@test "keepMin: 2 commits × 3 types, keepMin=5 → all kept (2 < 5)" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}}
+EOF
+
+  KEEP=()
+  KEEP+=("niko/r/reports/c1/test-a|niko|r|main|30")
+  KEEP+=("niko/r/reports/c1/test-b|niko|r|main|30")
+  KEEP+=("niko/r/reports/c1/test-c|niko|r|main|30")
+  KEEP+=("niko/r/reports/c2/test-a|niko|r|main|10")
+  KEEP+=("niko/r/reports/c2/test-b|niko|r|main|10")
+  KEEP+=("niko/r/reports/c2/test-c|niko|r|main|10")
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  [ "${#TO_DELETE[@]}" -eq 0 ]
+}
+
+@test "keepMin: 6 commits × 2 types, keepMin=3 → keeps 3 newest, deletes 3 oldest" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}}
+EOF
+
+  KEEP=()
+  local -a commits=(c1 c2 c3 c4 c5 c6)
+  local -a ages=(60 50 40 30 20 10)
+  local -a types=(jest pytest)
+  for i in "${!commits[@]}"; do
+    for t in "${types[@]}"; do
+      KEEP+=("niko/r/reports/${commits[$i]}/$t|niko|r|feature/x|${ages[$i]}")
+    done
+  done
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # Sort by days ascending: c6(10d), c5(20d), c4(30d), c3(40d), c2(50d), c1(60d)
+  # keepMin=3: c6,c5,c4 kept; c3,c2,c1 deleted
+  [ "${#TO_DELETE[@]}" -eq 3 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]]
+  [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]]
+  [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]]
+}
+
+# ---------------------------------------------------------------------------
+# apply_retention — maxAge
+# ---------------------------------------------------------------------------
+
+@test "maxAge: report exceeding maxAge is deleted" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}}
+EOF
+
+  KEEP=(
+    "niko/r/reports/c1/test|niko|r|main|100"
+    "niko/r/reports/c2/test|niko|r|main|50"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # c1 (100d) > 90, deleted; c2 (50d) < 90, kept
+  [ "${#TO_DELETE[@]}" -eq 1 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test" ]]
+}
+
+@test "maxAge deletes report-level dir, not commit-level" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":30,"keepMin":5}}}
+EOF
+
+  KEEP=(
+    "niko/r/reports/c1/test-a|niko|r|main|100"
+    "niko/r/reports/c1/test-b|niko|r|main|20"
+    "niko/r/reports/c2/test-a|niko|r|main|10"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # Only test-a for c1 is old; test-b for c1 is young, c2 is young
+  [ "${#TO_DELETE[@]}" -eq 1 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test-a" ]]
+}
+
+# ---------------------------------------------------------------------------
+# apply_retention — maxAge + keepMin interaction
+# ---------------------------------------------------------------------------
+
+@test "maxAge takes precedence over keepMin — aged report deleted, not counted in keepMin" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":30,"keepMin":2}}}
+EOF
+
+  # 3 commits, 1 type each. c1 is old (100d), c2 and c3 are young.
+  # With keepMin=2: c1 should be deleted by maxAge, c2 and c3 kept.
+  # Without the continue after maxAge check, c1 would consume a keepMin slot.
+  KEEP=(
+    "niko/r/reports/c1/test|niko|r|main|100"
+    "niko/r/reports/c2/test|niko|r|main|10"
+    "niko/r/reports/c3/test|niko|r|main|5"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # c1 deleted by maxAge, c2 and c3 within keepMin=2
+  [ "${#TO_DELETE[@]}" -eq 1 ]
+}
+
+# ---------------------------------------------------------------------------
+# apply_retention — sorting (newest first)
+# ---------------------------------------------------------------------------
+
+@test "sort order: newest commits processed first within same branch" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":365,"keepMin":2}}}
+EOF
+
+  KEEP=(
+    "niko/r/reports/c1/test|niko|r|main|100"
+    "niko/r/reports/c2/test|niko|r|main|50"
+    "niko/r/reports/c3/test|niko|r|main|10"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # Sort by days ascending: c3(10d) 1st, c2(50d) 2nd, c1(100d) 3rd
+  # keepMin=2: c3 and c2 kept, c1 deleted
+  [ "${#TO_DELETE[@]}" -eq 1 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]]
+}
+
+@test "sort order: branches sorted alphabetically" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":365,"keepMin":1}}}
+EOF
+
+  KEEP=(
+    "niko/r/reports/c1/test|niko|r|z-branch|50"
+    "niko/r/reports/c2/test|niko|r|a-branch|60"
+    "niko/r/reports/c3/test|niko|r|m-branch|10"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+
+  # Alphabetical: a-branch, m-branch, z-branch
+  # Each has 1 commit, keepMin=1 → nothing deleted
+  [ "${#TO_DELETE[@]}" -eq 0 ]
+}
+
+@test "multi-branch: each branch has own keepMin counter" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":2}}}
+EOF
+
+  KEEP=(
+    "niko/r/reports/c1/test|niko|r|branch-a|30"
+    "niko/r/reports/c2/test|niko|r|branch-a|20"
+    "niko/r/reports/c3/test|niko|r|branch-a|10"
+    "niko/r/reports/c4/test|niko|r|branch-b|60"
+    "niko/r/reports/c5/test|niko|r|branch-b|50"
+    "niko/r/reports/c6/test|niko|r|branch-b|40"
+    "niko/r/reports/c7/test|niko|r|branch-b|30"
+  )
+
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+ 
+  # branch-a: 3 reports → keep 2 newest (c2,c3), delete 1 oldest (c1)
+  # branch-b: 4 reports → keep 2 newest (c6,c7), delete 2 oldest (c4,c5)
+  # Actually: Sort is by branch, then by days ascending
+  # branch-a processed first: c3(10d) 1st, c2(20d) 2nd (keep), c1(30d) 3rd (delete)
+  # branch-b processed next: c7(30d) 1st, c6(40d) 2nd (keep), c5(50d) 3rd (delete), c4(60d) 4th (delete)
+  [ "${#TO_DELETE[@]}" -eq 3 ]
+  [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]]
+  [[ "${TO_DELETE[1]}" == "niko/r/reports/c5" ]]
+  [[ "${TO_DELETE[2]}" == "niko/r/reports/c4" ]]
+}
+
+# ---------------------------------------------------------------------------
+# apply_retention — empty / edge cases
+# ---------------------------------------------------------------------------
+
+@test "empty KEEP array → nothing deleted" {
+  write_config <<'EOF'
+{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}}
+EOF
+
+  KEEP=()
+  TO_DELETE=()
+  apply_retention "$CONFIG"
+  [ "${#TO_DELETE[@]}" -eq 0 ]
+}