From e75b425294c00cb93c25076f2b08d1e124e655d0 Mon Sep 17 00:00:00 2001 From: moilanik Date: Fri, 26 Jun 2026 05:31:25 +0300 Subject: [PATCH] retention korjaus testien avulla --- git-pages/README.md | 22 ++ git-pages/files/retention-cleanup.sh | 57 +--- git-pages/files/retention-lib.sh | 86 +++++ git-pages/templates/retention-configmap.yaml | 2 + git-pages/tests/retention.bats | 328 +++++++++++++++++++ 5 files changed, 441 insertions(+), 54 deletions(-) create mode 100644 git-pages/files/retention-lib.sh create mode 100644 git-pages/tests/retention.bats diff --git a/git-pages/README.md b/git-pages/README.md index 701eef4..e8f227c 100644 --- a/git-pages/README.md +++ b/git-pages/README.md @@ -149,3 +149,25 @@ curl -X PATCH https://ci-reports.helm-dev.keskikuja.site/owner/repo/commit/sha8/ - `git-pages-publish-token` = plaintext (luetaan Giteaan viedessä) Tarkemmat secret-ohjeet: [docs/secrets.md](docs/secrets.md). + +--- + +## Testaus + +Retention-logiikalle on unit-testit, jotka testaa funktiot ja Phase 3 -säännöt +erikseen ilman ulkoisia riippuvuuksia. + +```bash +cd git-pages +bats tests/retention.bats +``` + +Testit käyttävät `/files/retention-lib.sh` -jaettua kirjastoa, jota myös +`retention-cleanup.sh` sourceaa. Uutta testiä kirjoittaessa: + +1. Luo config `write_config`-helperilla +2. Täytä `KEEP`-array testidatalla (muoto: `dir|owner|repo|branch|days`) +3. Kutsu `apply_retention "$CONFIG"` +4. Tarkista `TO_DELETE`-array ja `$output` + +**Vaatimukset:** `bats`, `jq`, `date` (GNU date tai BSD date ISO 8601 -tuella). diff --git a/git-pages/files/retention-cleanup.sh b/git-pages/files/retention-cleanup.sh index 653fd6b..e3f69bd 100644 --- a/git-pages/files/retention-cleanup.sh +++ b/git-pages/files/retention-cleanup.sh @@ -53,35 +53,8 @@ branch_exists() { return 0 } -default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$CONFIG") -default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$CONFIG") - -rule_max_age() { - local branch="$1" v - v=$(jq -r --arg b "$branch" '.branches[$b].maxAgeDays // empty' "$CONFIG") - [ -n "$v" ] && echo "$v" || echo "$default_max_age" -} - -rule_keep_min() { - local branch="$1" v - v=$(jq -r --arg b "$branch" '.branches[$b].keepMin // empty' "$CONFIG") - [ -n "$v" ] && echo "$v" || echo "$default_keep_min" -} - -age_days() { - local published="$1" epoch_pub now - epoch_pub=$(date -u -d "$published" +%s 2>/dev/null || echo 0) - [ "$epoch_pub" -eq 0 ] && echo 99999 && return - now=$(date -u +%s) - echo $(( (now - epoch_pub) / 86400 )) -} - -parse_path() { - local rel="$1" - OWNER="${rel%%/*}" - rest="${rel#*/}" - REPO="${rest%%/*}" -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/retention-lib.sh" echo "Fetching manifest from ${PAGES_URL}/.git-pages/manifest.json" MANIFEST=$(curl_with_host "${PAGES_URL}/.git-pages/manifest.json") @@ -144,31 +117,7 @@ done echo "" echo "=== Phase 3: apply retention rules to remaining reports ===" -declare -A BRANCH_COUNTS -if [ "${#KEEP[@]}" -gt 0 ]; then - IFS=$'\n' - for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5rn); do - IFS='|' read -r dir owner repo branch days <<< "$entry" - max_age=$(rule_max_age "$branch") - keep_min=$(rule_keep_min "$branch") - - if [ "$days" -gt "$max_age" ]; then - echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})" - TO_DELETE+=("$dir") - continue - fi - - key="${branch}" - count="${BRANCH_COUNTS[$key]:-0}" - count=$((count + 1)) - BRANCH_COUNTS["$key"]=$count - if [ "$count" -gt "$keep_min" ]; then - echo " DELETE: ${dir} (kept ${keep_min}/${count}, exceeds keepMin, branch ${branch})" - TO_DELETE+=("$dir") - fi - done - unset IFS -fi +apply_retention "$CONFIG" if [ "${#TO_DELETE[@]}" -eq 0 ]; then echo "Nothing to delete" diff --git a/git-pages/files/retention-lib.sh b/git-pages/files/retention-lib.sh new file mode 100644 index 0000000..bf5f589 --- /dev/null +++ b/git-pages/files/retention-lib.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Shared functions for retention-cleanup.sh +# Can be sourced by tests for unit testing + +age_days() { + local published="$1" epoch_pub now + epoch_pub=$(date -d "$published" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$published" +%s 2>/dev/null || echo 0) + [ "$epoch_pub" -eq 0 ] && echo 99999 && return + now=$(date -u +%s) + echo $(( (now - epoch_pub) / 86400 )) +} + +parse_path() { + local rel="$1" + OWNER="${rel%%/*}" + rest="${rel#*/}" + REPO="${rest%%/*}" +} + +read_rule() { + local config="$1" branch="$2" key="$3" default="$4" + v=$(jq -r --arg b "$branch" --arg k "$key" '.branches[$b][$k] // empty' "$config") + [ -n "$v" ] && echo "$v" || echo "$default" +} + +# Phase 3: apply retention rules to KEEP array, populate TO_DELETE +# Reads from global KEEP array +# Populates global TO_DELETE array +# Usage: apply_retention +apply_retention() { + local config="$1" + local default_max_age default_keep_min + local max_age keep_min key count seen_key commit_dir report_type path + local entry dir owner repo branch days + + default_max_age=$(jq -r '.branches.default.maxAgeDays // 90' "$config") + default_keep_min=$(jq -r '.branches.default.keepMin // 5' "$config") + + TO_DELETE=() + declare -A BRANCH_COUNTS + declare -A SEEN_COMMITS + declare -A DELETED_COMMITS + + if [ "${#KEEP[@]}" -eq 0 ]; then + return + fi + + IFS=$'\n' + for entry in $(printf '%s\n' "${KEEP[@]}" | sort -t'|' -k4,4 -k5,5n); do + IFS='|' read -r dir owner repo branch days <<< "$entry" + + max_age=$(read_rule "$config" "$branch" "maxAgeDays" "$default_max_age") + keep_min=$(read_rule "$config" "$branch" "keepMin" "$default_keep_min") + + # Age check — per-report-type deletion + if [ "$days" -gt "$max_age" ]; then + echo " DELETE: ${dir} (age ${days}d > maxAge ${max_age}d, branch ${branch})" + TO_DELETE+=("$dir") + continue + fi + + # keepMin — per-commit counting + commit_dir=$(dirname "$dir") + key="$branch" + seen_key="${key}|${commit_dir}" + + if [ -z "${SEEN_COMMITS[$seen_key]:-}" ]; then + SEEN_COMMITS["$seen_key"]=1 + count="${BRANCH_COUNTS[$key]:-0}" + count=$((count + 1)) + BRANCH_COUNTS["$key"]=$count + else + count="${BRANCH_COUNTS[$key]:-0}" + fi + + if [ "$count" -gt "$keep_min" ]; then + if [ -z "${DELETED_COMMITS[$commit_dir]:-}" ]; then + DELETED_COMMITS[$commit_dir]=1 + report_type="${dir##*/}" + echo " DELETE: ${commit_dir} (kept ${keep_min}/${count} commits, exceeds keepMin, branch ${branch})" + TO_DELETE+=("$commit_dir") + fi + fi + done + unset IFS +} diff --git a/git-pages/templates/retention-configmap.yaml b/git-pages/templates/retention-configmap.yaml index 506728e..cc59680 100644 --- a/git-pages/templates/retention-configmap.yaml +++ b/git-pages/templates/retention-configmap.yaml @@ -8,6 +8,8 @@ metadata: data: retention.json: | {{- .Values.retention.rules | toJson | nindent 4 }} + retention-lib.sh: | + {{- .Files.Get "files/retention-lib.sh" | nindent 4 }} retention-cleanup.sh: | {{- .Files.Get "files/retention-cleanup.sh" | nindent 4 }} retention-run.sh: | diff --git a/git-pages/tests/retention.bats b/git-pages/tests/retention.bats new file mode 100644 index 0000000..25fbc3f --- /dev/null +++ b/git-pages/tests/retention.bats @@ -0,0 +1,328 @@ +#!/usr/bin/env bats + +setup() { + # Copy the retention lib to a temp location so it's easier to source + # (bats runs each test in its own dir) + source "$(dirname "$BATS_TEST_DIRNAME")/files/retention-lib.sh" + CONFIG=$(mktemp) +} + +teardown() { + rm -f "$CONFIG" +} + +write_config() { + cat > "$CONFIG" +} + +# --------------------------------------------------------------------------- +# read_rule +# --------------------------------------------------------------------------- + +@test "read_rule returns default when branch has no override" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + result=$(read_rule "$CONFIG" "nonexistent" "maxAgeDays" 90) + [ "$result" = "90" ] +} + +@test "read_rule returns branch-specific value" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365,"keepMin":20}}} +EOF + result=$(read_rule "$CONFIG" "main" "keepMin" 5) + [ "$result" = "20" ] +} + +@test "read_rule returns default for undefined key even if branch exists" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5},"main":{"maxAgeDays":365}}} +EOF + result=$(read_rule "$CONFIG" "main" "keepMin" 5) + [ "$result" = "5" ] +} + +# --------------------------------------------------------------------------- +# parse_path +# --------------------------------------------------------------------------- + +@test "parse_path extracts owner and repo" { + parse_path "my-owner/my-repo/reports/abc123/go-test-unit" + [ "$OWNER" = "my-owner" ] + [ "$REPO" = "my-repo" ] +} + +@test "parse_path handles owner with hyphens" { + parse_path "niko/agent-platform/reports/abc1234/go-test-bdd" + [ "$OWNER" = "niko" ] + [ "$REPO" = "agent-platform" ] +} + +# --------------------------------------------------------------------------- +# apply_retention — keepMin per commit +# --------------------------------------------------------------------------- + +@test "keepMin: 6 commits × 4 types, keepMin=10 → all kept (6 commits < 10)" { + # With old per-file counting, 24 files > 10 keepMin would delete 14. + # With per-commit counting, 6 commits < 10 keepMin keeps everything. + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":10}}} +EOF + + KEEP=() + local -a commits=(c1 c2 c3 c4 c5 c6) + local -a ages=(100 80 60 40 20 5) + local -a types=(go-test-bdd go-test-unit helm-lint helm-kubeconform) + for i in "${!commits[@]}"; do + for t in "${types[@]}"; do + KEEP+=("niko/agent-platform/reports/${commits[$i]}/$t|niko|agent-platform|main|${ages[$i]}") + done + done + + TO_DELETE=() + apply_retention "$CONFIG" + + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "keepMin: 8 commits × 1 type, keepMin=5 → deletes 3 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":5}}} +EOF + + KEEP=() + local -a ages=(80 70 60 50 40 30 20 10) + for i in "${!ages[@]}"; do + KEEP+=("niko/r/reports/c$((i+1))/test|niko|r|main|${ages[$i]}") + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # 5 newest (c8-c4) kept, 3 oldest (c3,c2,c1) deleted + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]] +} + +@test "keepMin: 12 commits × 1 type, keepMin=5 → keeps 5 newest, deletes 7 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + for i in $(seq 1 12); do + KEEP+=("niko/r/reports/c${i}/test|niko|r|feature/foo|$(( 13 - i ))") + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # 7 oldest commits deleted (12 - 5 = 7) + [ "${#TO_DELETE[@]}" -eq 7 ] + # Oldest 7 should be c1..c7 (highest days = oldest = processed last after sort) + # Sort is ascending by days, so processed as c12(1d), c11(2d), ..., c1(12d) + # keepMin=5: c12-c8 kept, c7-c1 deleted + [[ "${TO_DELETE[0]}" == "niko/r/reports/c7" ]] + [[ "${TO_DELETE[6]}" == "niko/r/reports/c1" ]] +} + +@test "keepMin: 2 commits × 3 types, keepMin=5 → all kept (2 < 5)" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + KEEP+=("niko/r/reports/c1/test-a|niko|r|main|30") + KEEP+=("niko/r/reports/c1/test-b|niko|r|main|30") + KEEP+=("niko/r/reports/c1/test-c|niko|r|main|30") + KEEP+=("niko/r/reports/c2/test-a|niko|r|main|10") + KEEP+=("niko/r/reports/c2/test-b|niko|r|main|10") + KEEP+=("niko/r/reports/c2/test-c|niko|r|main|10") + + TO_DELETE=() + apply_retention "$CONFIG" + + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "keepMin: 6 commits × 2 types, keepMin=3 → keeps 3 newest, deletes 3 oldest" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":3}}} +EOF + + KEEP=() + local -a commits=(c1 c2 c3 c4 c5 c6) + local -a ages=(60 50 40 30 20 10) + local -a types=(jest pytest) + for i in "${!commits[@]}"; do + for t in "${types[@]}"; do + KEEP+=("niko/r/reports/${commits[$i]}/$t|niko|r|feature/x|${ages[$i]}") + done + done + + TO_DELETE=() + apply_retention "$CONFIG" + + # Sort by days ascending: c6(10d), c5(20d), c4(30d), c3(40d), c2(50d), c1(60d) + # keepMin=3: c6,c5,c4 kept; c3,c2,c1 deleted + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c3" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c2" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c1" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — maxAge +# --------------------------------------------------------------------------- + +@test "maxAge: report exceeding maxAge is deleted" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|50" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # c1 (100d) > 90, deleted; c2 (50d) < 90, kept + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test" ]] +} + +@test "maxAge deletes report-level dir, not commit-level" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":30,"keepMin":5}}} +EOF + + KEEP=( + "niko/r/reports/c1/test-a|niko|r|main|100" + "niko/r/reports/c1/test-b|niko|r|main|20" + "niko/r/reports/c2/test-a|niko|r|main|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Only test-a for c1 is old; test-b for c1 is young, c2 is young + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1/test-a" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — maxAge + keepMin interaction +# --------------------------------------------------------------------------- + +@test "maxAge takes precedence over keepMin — aged report deleted, not counted in keepMin" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":30,"keepMin":2}}} +EOF + + # 3 commits, 1 type each. c1 is old (100d), c2 and c3 are young. + # With keepMin=2: c1 should be deleted by maxAge, c2 and c3 kept. + # Without the continue after maxAge check, c1 would consume a keepMin slot. + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|10" + "niko/r/reports/c3/test|niko|r|main|5" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # c1 deleted by maxAge, c2 and c3 within keepMin=2 + [ "${#TO_DELETE[@]}" -eq 1 ] +} + +# --------------------------------------------------------------------------- +# apply_retention — sorting (newest first) +# --------------------------------------------------------------------------- + +@test "sort order: newest commits processed first within same branch" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":2}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|main|100" + "niko/r/reports/c2/test|niko|r|main|50" + "niko/r/reports/c3/test|niko|r|main|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Sort by days ascending: c3(10d) 1st, c2(50d) 2nd, c1(100d) 3rd + # keepMin=2: c3 and c2 kept, c1 deleted + [ "${#TO_DELETE[@]}" -eq 1 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]] +} + +@test "sort order: branches sorted alphabetically" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":365,"keepMin":1}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|z-branch|50" + "niko/r/reports/c2/test|niko|r|a-branch|60" + "niko/r/reports/c3/test|niko|r|m-branch|10" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # Alphabetical: a-branch, m-branch, z-branch + # Each has 1 commit, keepMin=1 → nothing deleted + [ "${#TO_DELETE[@]}" -eq 0 ] +} + +@test "multi-branch: each branch has own keepMin counter" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":2}}} +EOF + + KEEP=( + "niko/r/reports/c1/test|niko|r|branch-a|30" + "niko/r/reports/c2/test|niko|r|branch-a|20" + "niko/r/reports/c3/test|niko|r|branch-a|10" + "niko/r/reports/c4/test|niko|r|branch-b|60" + "niko/r/reports/c5/test|niko|r|branch-b|50" + "niko/r/reports/c6/test|niko|r|branch-b|40" + "niko/r/reports/c7/test|niko|r|branch-b|30" + ) + + TO_DELETE=() + apply_retention "$CONFIG" + + # branch-a: 3 reports → keep 2 newest (c2,c3), delete 1 oldest (c1) + # branch-b: 4 reports → keep 2 newest (c6,c7), delete 2 oldest (c4,c5) + # Actually: Sort is by branch, then by days ascending + # branch-a processed first: c3(10d) 1st, c2(20d) 2nd (keep), c1(30d) 3rd (delete) + # branch-b processed next: c7(30d) 1st, c6(40d) 2nd (keep), c5(50d) 3rd (delete), c4(60d) 4th (delete) + [ "${#TO_DELETE[@]}" -eq 3 ] + [[ "${TO_DELETE[0]}" == "niko/r/reports/c1" ]] + [[ "${TO_DELETE[1]}" == "niko/r/reports/c5" ]] + [[ "${TO_DELETE[2]}" == "niko/r/reports/c4" ]] +} + +# --------------------------------------------------------------------------- +# apply_retention — empty / edge cases +# --------------------------------------------------------------------------- + +@test "empty KEEP array → nothing deleted" { + write_config <<'EOF' +{"branches":{"default":{"maxAgeDays":90,"keepMin":5}}} +EOF + + KEEP=() + TO_DELETE=() + apply_retention "$CONFIG" + [ "${#TO_DELETE[@]}" -eq 0 ] +}