test(ingest): Add tests for slug validation and index-append updates

skills/ingest/scripts/index-append.py

@@ -18,6 +18,7 @@ import re
 import sys
 
 ENTRY_RE = re.compile(r"^- \[\[")
+LINK_RE = re.compile(r"^- \[\[([^\]]+)\]\]")
 HEADER_RE = re.compile(r"^## ")
 
 
@@ -49,6 +50,10 @@ def main() -> int:
         if fm_open and ln.startswith("last_updated:"):
             lines[i] = f"last_updated: {today}"
 
+    if not fm_open:
+        print("index-append: warning: no frontmatter found, last_updated not bumped",
+              file=sys.stderr)
+
     # 2. Locate the target section [start, end)
     start = None
     for i, ln in enumerate(lines):
@@ -71,11 +76,31 @@ def main() -> int:
     intro = [ln for ln in body if not ENTRY_RE.match(ln)]
     entries = [ln for ln in body if ENTRY_RE.match(ln)]
 
-    if args.entry in entries:
+    # Deduplicate by wikilink PATH, not by exact line: a re-ingest with a changed
-        print(f"index-append: entry already present, skipping")
+    # summary/maturity should UPDATE the existing entry, not add a duplicate line.
-        return 0
+    new_m = LINK_RE.match(args.entry)
+    new_link = new_m.group(1) if new_m else None
+
+    if new_link is not None:
+        replaced = False
+        for idx, ln in enumerate(entries):
+            m = LINK_RE.match(ln)
+            if m and m.group(1) == new_link:
+                if ln == args.entry:
+                    print("index-append: entry already present, skipping")
+                    return 0
+                entries[idx] = args.entry          # same page, refreshed text
+                replaced = True
+                break
+        if not replaced:
+            entries.append(args.entry)
+    else:
+        # No parseable wikilink — fall back to exact-line dedup.
+        if args.entry in entries:
+            print("index-append: entry already present, skipping")
+            return 0
+        entries.append(args.entry)
 
-    entries.append(args.entry)
     entries.sort(key=str.casefold)
 
     # Normalise intro: drop trailing blanks, keep header + comment(s)

skills/ingest/scripts/log-append.sh

@@ -35,7 +35,7 @@ esac
 
 [[ -f "$LOG_FILE" ]] || { echo "log-append: not found: $LOG_FILE" >&2; exit 1; }
 
-run_id="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"
+run_id="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c 'import uuid; print(uuid.uuid4())')"
 today="$(date +%Y-%m-%d)"
 
 {

skills/ingest/scripts/open-pr.sh

@@ -39,11 +39,13 @@ repo="$(basename -s .git "$(git config --get remote.origin.url)")"
 # 1. Branch + commit + push (AGENTS.md rule 5: never commit to main)
 git switch -c "$branch" 2>/dev/null || git switch "$branch"
 git add wiki/
-if git diff --cached --quiet; then
+# Scope BOTH the emptiness check and the commit to wiki/ — never commit anything that
+# happened to be staged outside wiki/ (a stray hook, an aborted prior run, etc.).
+if git diff --cached --quiet -- wiki/; then
   echo "open-pr: nothing staged under wiki/ — aborting" >&2
   exit 1
 fi
-git commit -m "$title"
+git commit -m "$title" -- wiki/
 git push -u origin "$branch"
 
 # DRY_RUN: local git work done; skip the Forgejo API (offline tests).
@@ -53,19 +55,23 @@ if [[ -n "${DRY_RUN:-}" ]]; then
 fi
 
 # 2. Open the PR via Forgejo API (jq builds the JSON safely)
+# TODO: Forgejo-only. When registry.sh/globals.env sets PROVIDER=github, branch on
+# $PROVIDER here and delegate to providers/github.sh (same token + http_code contract).
 body="$(cat "$body_file")"
 payload="$(jq -n --arg head "$branch" --arg base "$base" \
                  --arg title "$title" --arg body "$body" \
                  '{head:$head, base:$base, title:$title, body:$body}')"
 
-resp="$(curl -s -w '\n%{http_code}' \
+resp="$(curl --max-time 30 -s -w '\n%{http_code}' \
   -H "Authorization: token ${FORGEJO_TOKEN}" \
   -H "Content-Type: application/json" \
   -X POST "${FORGEJO_URL}/api/v1/repos/${FORGEJO_USER}/${repo}/pulls" \
   -d "$payload")"
 
-code="$(printf '%s' "$resp" | tail -n1)"
+# curl -w appends '\n<code>' AFTER the body, so the code is always the final line and the
-json="$(printf '%s' "$resp" | sed '$d')"
+# body is everything before it. Parameter expansion (no subshells), robust to multi-line JSON.
+code="${resp##*$'\n'}"
+json="${resp%$'\n'*}"
 
 case "$code" in
   201)
@@ -89,11 +95,11 @@ esac
 
 # 3. Optional label (e.g. CONFLICT). Best-effort; non-fatal.
 if [[ -n "$label" && -n "${number:-}" ]]; then
-  label_id="$(curl -s -H "Authorization: token ${FORGEJO_TOKEN}" \
+  label_id="$(curl --max-time 15 -s -H "Authorization: token ${FORGEJO_TOKEN}" \
     "${FORGEJO_URL}/api/v1/repos/${FORGEJO_USER}/${repo}/labels" \
     | jq -r --arg n "$label" '.[] | select(.name==$n) | .id' | head -n1)"
   if [[ -n "$label_id" && "$label_id" != "null" ]]; then
-    curl -s -o /dev/null \
+    curl --max-time 15 -s -o /dev/null \
       -H "Authorization: token ${FORGEJO_TOKEN}" -H "Content-Type: application/json" \
       -X POST "${FORGEJO_URL}/api/v1/repos/${FORGEJO_USER}/${repo}/issues/${number}/labels" \
       -d "{\"labels\":[${label_id}]}" \

skills/ingest/scripts/run-ingest.sh

@@ -36,7 +36,7 @@ contradictions="$(jq -r '.contradictions // "None"' "$manifest")"
 
 [[ -n "$raw_source" && "$raw_source" != "null" ]] || fail "manifest" "raw_source missing"
 
-slug="$(bash "${SCRIPTS}/slug.sh" "$raw_source")"
+slug="$(bash "${SCRIPTS}/slug.sh" "$raw_source")" || fail "slug" "empty or invalid slug for ${raw_source}"
 
 # --- collect touched paths ---
 mapfile -t created_paths  < <(jq -r '.pages[] | select(.status=="created")  | .path' "$manifest")
@@ -46,6 +46,11 @@ all_paths=( "${created_paths[@]}" "${modified_paths[@]}" )
 
 conflict_label=""
 
+# NOTE: no rollback. Steps below mutate the working tree in order (index → log → commit).
+# All are idempotent on re-run EXCEPT log-append (append-only). If a step fails midway,
+# nothing is committed (open-pr is the only committer) — the operator re-runs, or inspects
+# wiki/ if log-append already wrote a line. The manifest is removed only on full success.
+
 # --- 1. index entries (created pages only), inserted in order ---
 while IFS=$'\t' read -r path summary maturity; do
   [[ -z "$path" ]] && continue
@@ -119,4 +124,10 @@ jq -nc \
   --arg detail   "$pr_out" \
   '{status:$status, slug:$slug, pr_url:$pr_url, lint_clean:$lint_clean, conflict:$conflict, detail:$detail}'
 
-[[ $pr_rc -eq 0 ]]
+# The manifest is a single file overwritten by each pi run (not accumulating), but on full
+# success we remove it so a stale manifest can never be re-processed by mistake.
+if [[ $pr_rc -eq 0 ]]; then
+  rm -f "$manifest"
+else
+  exit 1
+fi

skills/ingest/scripts/scoped-lint.sh

@@ -12,7 +12,12 @@
 # =============================================================================
 set -euo pipefail
 
-: "${KG_LIB_DIR:?set KG_LIB_DIR to the framework lib/ dir (e.g. /opt/knowledge-genome-setup/lib)}"
+: "${KG_LIB_DIR:?set KG_LIB_DIR to the framework lib/ dir (e.g. /opt/knowledge-genome-orchestrator/lib)}"
+
+# Fail clearly if the lib files are missing, rather than a raw `source: No such file`.
+for _f in output.sh lint.sh; do
+  [[ -f "${KG_LIB_DIR}/${_f}" ]] || { echo "scoped-lint: missing ${KG_LIB_DIR}/${_f}" >&2; exit 1; }
+done
 
 # shellcheck source=/dev/null
 source "${KG_LIB_DIR}/output.sh"

skills/ingest/scripts/slug.sh

@@ -13,6 +13,11 @@ input="${1:?usage: slug.sh <path-or-title>}"
 base="${input##*/}"
 base="${base%.*}"
 
-printf '%s\n' "$base" \
+slug="$(printf '%s\n' "$base" \
   | tr '[:upper:]' '[:lower:]' \
-  | sed -E 's/[^a-z0-9]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//'
+  | sed -E 's/[^a-z0-9]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//')"
+
+# An all-symbols input (e.g. "!!!.md") collapses to "" — refuse rather than emit a
+# broken/empty slug that would produce an invalid branch name downstream.
+[[ -n "$slug" ]] || { echo "slug: empty result for input '${input}'" >&2; exit 1; }
+printf '%s\n' "$slug"

tests/scripts.bats

@@ -51,3 +51,18 @@ load helpers
   python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/dup]] — d. `maturity: draft`'
   [ "$(grep -c 'sources/dup' wiki/index.md)" -eq 1 ]
 }
+
+@test "index-append: updates an existing entry by wikilink path (no duplicate)" {
+  G="$(make_fixture_genome)"; cd "$G"
+  python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/foo]] — old summary. `maturity: draft`'
+  python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/foo]] — new summary. `maturity: stable`'
+  [ "$(grep -c 'sources/foo' wiki/index.md)" -eq 1 ]
+  grep -q 'new summary' wiki/index.md
+  ! grep -q 'old summary' wiki/index.md
+}
+
+@test "slug: refuses an all-symbols input (no empty slug)" {
+  run bash "$SKILL_SCRIPTS/slug.sh" "!!!.md"
+  [ "$status" -ne 0 ]
+  [ -z "$output" ] || [[ "$output" != *"feat/ai-ingest-"* ]]
+}