Compare commits
11 commits
95b3866549
...
0fd70e257b
| Author | SHA1 | Date | |
|---|---|---|---|
| 0fd70e257b | |||
| f1a682850f | |||
| 591883af47 | |||
| 865fdb95f4 | |||
| c4aba8507c | |||
| 1c6d7a4ecd | |||
| 066db00e89 | |||
| c0659d5ce9 | |||
| 101eef98aa | |||
| 8082bc3003 | |||
| 990118de71 |
10 changed files with 831 additions and 93 deletions
322
deploy/n8n/genome-prune.json
Normal file
322
deploy/n8n/genome-prune.json
Normal file
|
|
@ -0,0 +1,322 @@
|
||||||
|
{
|
||||||
|
"name": "Genome: prune",
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"httpMethod": "POST",
|
||||||
|
"path": "forgejo-push-prune",
|
||||||
|
"options": {}
|
||||||
|
},
|
||||||
|
"id": "050b318b-d3bb-4c1a-baa4-a1e2bd1babd8",
|
||||||
|
"name": "Webhook prune",
|
||||||
|
"type": "n8n-nodes-base.webhook",
|
||||||
|
"typeVersion": 2.1,
|
||||||
|
"position": [
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"webhookId": "d6ac11900058434e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"jsCode": "// Bell filter for PRUNING: proceed only on develop pushes that REMOVED a raw/ file.\n// Adds/modifications are the ingest flow's job; this flow reacts to deletions only.\nconst b = $json.body || $json;\nconst ref = b.ref || '';\nconst genome = (b.repository && b.repository.name) || '';\nif (ref !== 'refs/heads/develop') return [];\nif (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(genome)) return [];\nconst removed = [];\nfor (const c of (b.commits || [])) for (const p of (c.removed || [])) removed.push(p);\nif (!removed.some(p => p.startsWith('raw/'))) return []; // nothing under raw/ removed -> ignore\nreturn [{ json: { genome } }];"
|
||||||
|
},
|
||||||
|
"id": "deac740a-2046-4de8-91eb-812114edeb7b",
|
||||||
|
"name": "Gate prune",
|
||||||
|
"type": "n8n-nodes-base.code",
|
||||||
|
"typeVersion": 2,
|
||||||
|
"position": [
|
||||||
|
224,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"workflowId": {
|
||||||
|
"__rl": true,
|
||||||
|
"value": "zbtRXWsLt56nEIfz",
|
||||||
|
"mode": "list",
|
||||||
|
"cachedResultUrl": "/workflow/zbtRXWsLt56nEIfz",
|
||||||
|
"cachedResultName": "Power Manager"
|
||||||
|
},
|
||||||
|
"workflowInputs": {
|
||||||
|
"mappingMode": "defineBelow",
|
||||||
|
"value": {
|
||||||
|
"mode": "ensure-on"
|
||||||
|
},
|
||||||
|
"matchingColumns": [
|
||||||
|
"mode"
|
||||||
|
],
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"id": "mode",
|
||||||
|
"displayName": "mode",
|
||||||
|
"required": false,
|
||||||
|
"defaultMatch": false,
|
||||||
|
"display": true,
|
||||||
|
"canBeUsedToMatch": true,
|
||||||
|
"type": "string",
|
||||||
|
"removed": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"attemptToConvertTypes": false,
|
||||||
|
"convertFieldsToString": true
|
||||||
|
},
|
||||||
|
"options": {}
|
||||||
|
},
|
||||||
|
"id": "554737ea-09f3-4d4b-9a66-c983cc72e655",
|
||||||
|
"name": "Power Manager - ensure-on",
|
||||||
|
"type": "n8n-nodes-base.executeWorkflow",
|
||||||
|
"typeVersion": 1.3,
|
||||||
|
"position": [
|
||||||
|
448,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"authentication": "privateKey",
|
||||||
|
"command": "=ssh vm101 'pi orphan-wiki {{ $('Gate prune').first().json.genome }}'"
|
||||||
|
},
|
||||||
|
"id": "311005a4-16fd-4752-92d3-e3bbb9cdf19f",
|
||||||
|
"name": "SSH: orphan-wiki",
|
||||||
|
"type": "n8n-nodes-base.ssh",
|
||||||
|
"typeVersion": 1,
|
||||||
|
"position": [
|
||||||
|
672,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"credentials": {
|
||||||
|
"sshPrivateKey": {
|
||||||
|
"id": "GJQjKzte7Hjdfz89",
|
||||||
|
"name": "n8n container -> n8n-runner@nexus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"jsCode": "// Gate: prune only if orphan-wiki found orphans. run-prune re-derives independently anyway\n// (no detected-vs-pruned race) — this just avoids taking the lock for nothing.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d; try { d = JSON.parse(out); } catch (e) { return []; }\nif (!d || !d.count) return []; // 0 orphans -> stop silently\nreturn [{ json: { genome: d.genome, count: d.count } }];"
|
||||||
|
},
|
||||||
|
"id": "2374b63c-f0db-4ae9-b350-3fec70687384",
|
||||||
|
"name": "Orfani?",
|
||||||
|
"type": "n8n-nodes-base.code",
|
||||||
|
"typeVersion": 2,
|
||||||
|
"position": [
|
||||||
|
880,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"authentication": "privateKey",
|
||||||
|
"command": "=ssh vm101 'pi prune {{ $json.genome }}'"
|
||||||
|
},
|
||||||
|
"id": "e4130173-ff62-4e11-b3d1-ee7870803663",
|
||||||
|
"name": "SSH: prune",
|
||||||
|
"type": "n8n-nodes-base.ssh",
|
||||||
|
"typeVersion": 1,
|
||||||
|
"position": [
|
||||||
|
1104,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"credentials": {
|
||||||
|
"sshPrivateKey": {
|
||||||
|
"id": "GJQjKzte7Hjdfz89",
|
||||||
|
"name": "n8n container -> n8n-runner@nexus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"mode": "runOnceForEachItem",
|
||||||
|
"jsCode": "const out = ($json.stdout || '').toString().trim();\nconst line = out.split('\\n').filter(l => l.trim().startsWith('{')).pop();\nlet r; try { r = line ? JSON.parse(line) : { status:'error', reason:'nessuna riga JSON' }; }\ncatch (e) { r = { status:'error', reason:'JSON non parsabile' }; }\nreturn r;"
|
||||||
|
},
|
||||||
|
"id": "b649df9a-1e64-49cc-9e7f-6f78a1190382",
|
||||||
|
"name": "Parse prune",
|
||||||
|
"type": "n8n-nodes-base.code",
|
||||||
|
"typeVersion": 2,
|
||||||
|
"position": [
|
||||||
|
1328,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"mode": "runOnceForEachItem",
|
||||||
|
"jsCode": "// Pruning notification on genome-ingest (the \"system produced a PR to judge\" topic), broom icon.\nconst d = $json;\nconst g = $('Orfani?').first().json;\nlet n;\nif (d.status === 'ok') {\n const pm = (d.pr_url || '').match(/\\/pulls\\/(\\d+)/); const num = pm ? `#${pm[1]}` : '';\n n = { topic:'genome-ingest', title:`${g.genome} \\u00b7 potatura ${num}`.replace(/\\s+/g,' ').trim(),\n priority:'default', tags:'broom', click:d.pr_url || '', actions:d.pr_url ? `view, Apri la PR, ${d.pr_url}` : '',\n body:`${d.count} sorgente/i orfane proposte per la rimozione. **Approva la PR** per potare, oppure chiudila da Forgejo per annullare.` };\n} else {\n n = { topic:'genome-ingest', title:`${g ? g.genome : ''} \\u00b7 errore potatura`.trim(), priority:'high',\n tags:'rotating_light', click:'', actions:'', body:`${d.reason || 'errore'}.` };\n}\nreturn n;"
|
||||||
|
},
|
||||||
|
"id": "00be2a1f-6097-40e2-81f5-9dcba40b66ae",
|
||||||
|
"name": "Build ntfy",
|
||||||
|
"type": "n8n-nodes-base.code",
|
||||||
|
"typeVersion": 2,
|
||||||
|
"position": [
|
||||||
|
1552,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parameters": {
|
||||||
|
"method": "POST",
|
||||||
|
"url": "=http://ntfy/{{ $json.topic }}",
|
||||||
|
"authentication": "genericCredentialType",
|
||||||
|
"genericAuthType": "httpBearerAuth",
|
||||||
|
"sendHeaders": true,
|
||||||
|
"headerParameters": {
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "Title",
|
||||||
|
"value": "={{ $json.title }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Priority",
|
||||||
|
"value": "={{ $json.priority }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Tags",
|
||||||
|
"value": "={{ $json.tags }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Click",
|
||||||
|
"value": "={{ $json.click }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Actions",
|
||||||
|
"value": "={{ $json.actions }}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Markdown",
|
||||||
|
"value": "yes"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sendBody": true,
|
||||||
|
"contentType": "raw",
|
||||||
|
"rawContentType": "Raw / Text",
|
||||||
|
"body": "={{ $json.body }}",
|
||||||
|
"options": {
|
||||||
|
"timeout": 15000
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"id": "50a36840-be50-43ec-bea9-44819a88a923",
|
||||||
|
"name": "ntfy: send",
|
||||||
|
"type": "n8n-nodes-base.httpRequest",
|
||||||
|
"typeVersion": 4.4,
|
||||||
|
"position": [
|
||||||
|
1760,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"credentials": {
|
||||||
|
"httpHeaderAuth": {
|
||||||
|
"id": "TBPXSWOF63k9mvm8",
|
||||||
|
"name": "ntfy-token"
|
||||||
|
},
|
||||||
|
"httpBearerAuth": {
|
||||||
|
"id": "nCv4CUN7Ef086Ewj",
|
||||||
|
"name": "Bearer Auth account"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"pinData": {},
|
||||||
|
"connections": {
|
||||||
|
"Webhook prune": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "Gate prune",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Gate prune": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "Power Manager - ensure-on",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Power Manager - ensure-on": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "SSH: orphan-wiki",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"SSH: orphan-wiki": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "Orfani?",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Orfani?": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "SSH: prune",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"SSH: prune": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "Parse prune",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Parse prune": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "Build ntfy",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Build ntfy": {
|
||||||
|
"main": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"node": "ntfy: send",
|
||||||
|
"type": "main",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"active": true,
|
||||||
|
"settings": {
|
||||||
|
"executionOrder": "v1",
|
||||||
|
"binaryMode": "separate"
|
||||||
|
},
|
||||||
|
"versionId": "ff0be89b-7930-4171-a547-5dc7bffc9472",
|
||||||
|
"meta": {
|
||||||
|
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
|
||||||
|
},
|
||||||
|
"id": "smH5Qrv7CQnTtdAF",
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
|
@ -28,7 +28,8 @@ set -a; . "${HOME}/.config/knowledge-genome.env"; set +a
|
||||||
vault="${GENOME_VAULTS_ROOT}/${genome}"
|
vault="${GENOME_VAULTS_ROOT}/${genome}"
|
||||||
fid="${genome}-public"
|
fid="${genome}-public"
|
||||||
authors_map="${GENOME_VAULTS_ROOT}/.authors.json"
|
authors_map="${GENOME_VAULTS_ROOT}/.authors.json"
|
||||||
clone_url="http://${FORGEJO_USER}@${FORGEJO_HOST}/${FORGEJO_OWNER}/${genome}.git"
|
# GENOME_PUSH_URL is a test seam: defaults to the Forgejo loopback URL in production.
|
||||||
|
clone_url="${GENOME_PUSH_URL:-http://${FORGEJO_USER}@${FORGEJO_HOST}/${FORGEJO_OWNER}/${genome}.git}"
|
||||||
export GIT_ASKPASS=/usr/local/bin/genome-askpass
|
export GIT_ASKPASS=/usr/local/bin/genome-askpass
|
||||||
|
|
||||||
[[ -d "${vault}/.git" ]] || { printf '{"status":"error","reason":"vault absent","genome":"%s"}\n' "$genome"; exit 1; }
|
[[ -d "${vault}/.git" ]] || { printf '{"status":"error","reason":"vault absent","genome":"%s"}\n' "$genome"; exit 1; }
|
||||||
|
|
@ -42,8 +43,30 @@ grep -qxF 'raw/.stfolder' "${vault}/.git/info/exclude" 2>/dev/null || echo 'raw/
|
||||||
|
|
||||||
git add -A -- raw/
|
git add -A -- raw/
|
||||||
git reset -q -- raw/.stignore raw/.stfolder 2>/dev/null || true
|
git reset -q -- raw/.stignore raw/.stfolder 2>/dev/null || true
|
||||||
|
|
||||||
|
# --- Quiet window: only commit raw files that have STOPPED changing. ----------------
|
||||||
|
# While a note is being written (Obsidian autosave -> Syncthing -> here) its mtime stays
|
||||||
|
# fresh; we leave it UNSTAGED so a half-written note never triggers an ingest. A file is
|
||||||
|
# committed only after it has been still for RAW_QUIET_MINUTES. Deletions (nothing on disk)
|
||||||
|
# are stable by definition and pass straight through. Deterministic — no model in the loop.
|
||||||
|
quiet_min="${RAW_QUIET_MINUTES:-2}"
|
||||||
|
held=0
|
||||||
|
while IFS= read -r f; do
|
||||||
|
[[ -z "$f" ]] && continue
|
||||||
|
# Only an existing file can be "hot"; a staged deletion has nothing on disk to settle.
|
||||||
|
if [[ -e "$f" && -n "$(find "$f" -mmin -"$quiet_min" 2>/dev/null)" ]]; then
|
||||||
|
git reset -q -- "$f" 2>/dev/null || true
|
||||||
|
held=$((held+1))
|
||||||
|
fi
|
||||||
|
done < <(git diff --cached --name-only -- raw/)
|
||||||
|
|
||||||
if git diff --cached --quiet; then
|
if git diff --cached --quiet; then
|
||||||
printf '{"status":"noop","genome":"%s"}\n' "$genome"
|
if [[ "$held" -gt 0 ]]; then
|
||||||
|
printf '{"status":"noop","reason":"raw still settling","genome":"%s","held":%d,"quiet_minutes":%d}\n' \
|
||||||
|
"$genome" "$held" "$quiet_min"
|
||||||
|
else
|
||||||
|
printf '{"status":"noop","genome":"%s"}\n' "$genome"
|
||||||
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,29 @@ case "$cmd" in
|
||||||
# MECHANICAL step: validate manifest -> index/log/scoped-lint/commit/PR -> 1 JSON line
|
# MECHANICAL step: validate manifest -> index/log/scoped-lint/commit/PR -> 1 JSON line
|
||||||
exec "${HOME}/.pi/agent/skills/ingest/scripts/run-ingest.sh" "${genome}"
|
exec "${HOME}/.pi/agent/skills/ingest/scripts/run-ingest.sh" "${genome}"
|
||||||
;;
|
;;
|
||||||
|
"pi prune "*)
|
||||||
|
# Pota le source orfane. Stesso lock dell'ingest (serializza le scritture per genoma),
|
||||||
|
# clean_start, poi run-prune.sh (che ri-deriva gli orfani e apre una PR gated).
|
||||||
|
genome="${cmd#pi prune }"
|
||||||
|
case "$genome" in ""|*[!a-z0-9-]*) echo '{"status":"error","reason":"invalid genome name"}'; exit 1;; esac
|
||||||
|
logger -t n8n-pi-wrap "ok: pi prune ${genome}"
|
||||||
|
|
||||||
|
exec 9>"/run/lock/kg-ingest-${genome}.lock" 2>/dev/null || exec 9>"/tmp/kg-ingest-${genome}.lock"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo '{"status":"busy","reason":"another ingest/prune is running for this genome","genome":"'"$genome"'"}'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -a; . "${HOME}/.config/knowledge-genome.env"; set +a
|
||||||
|
cd "${GENOMES_ROOT}/${genome}" || { echo '{"status":"error","reason":"unknown genome"}'; exit 1; }
|
||||||
|
|
||||||
|
: "${KG_LIB_DIR:=${HOME}/knowledge-genome-orchestrator/lib}"
|
||||||
|
source "${KG_LIB_DIR}/clean-start.sh" 2>/dev/null \
|
||||||
|
|| { echo '{"status":"error","reason":"clean-start.sh not found"}'; exit 1; }
|
||||||
|
clean_start || { echo '{"status":"error","reason":"clean-start failed"}'; exit 1; }
|
||||||
|
|
||||||
|
exec "${HOME}/.pi/agent/skills/ingest/scripts/run-prune.sh" "${genome}"
|
||||||
|
;;
|
||||||
"pi ingest-rework "*)
|
"pi ingest-rework "*)
|
||||||
# args: <genome> <raw_path> <feedback_base64> (3 token).
|
# args: <genome> <raw_path> <feedback_base64> (3 token).
|
||||||
# Feedback in base64 nell'argv: il nodo SSH di n8n non passa stdin, e cosi' i metacaratteri
|
# Feedback in base64 nell'argv: il nodo SSH di n8n non passa stdin, e cosi' i metacaratteri
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# skills/ingest/scripts/index-append.py
|
# skills/ingest/scripts/index-append.py
|
||||||
# Insert an entry line into the correct section of wiki/index.md and keep that
|
# Insert OR remove an entry line in wiki/index.md, keeping the target section
|
||||||
# section's entries alphabetically ordered. Bumps frontmatter last_updated.
|
# alphabetically ordered. Bumps frontmatter last_updated.
|
||||||
#
|
#
|
||||||
# index-append.py --section Sources \
|
# index-append.py --section Sources \
|
||||||
# --entry '- [[sources/foo]] — One-line summary. `maturity: draft`'
|
# --entry '- [[sources/foo]] — One-line summary. `maturity: draft`'
|
||||||
|
# index-append.py --remove 'sources/foo' # delete the entry by wikilink
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
import argparse
|
import argparse
|
||||||
import datetime
|
import datetime
|
||||||
|
|
@ -17,14 +18,116 @@ LINK_RE = re.compile(r"^- \[\[([^\]]+)\]\]")
|
||||||
HEADER_RE = re.compile(r"^## ")
|
HEADER_RE = re.compile(r"^## ")
|
||||||
|
|
||||||
|
|
||||||
|
def bump_last_updated(lines, today):
|
||||||
|
"""Bump (or self-heal) last_updated inside the first frontmatter block."""
|
||||||
|
fm_open = False
|
||||||
|
fm_close_idx = None
|
||||||
|
bumped = False
|
||||||
|
for i, ln in enumerate(lines):
|
||||||
|
if ln.strip() == "---":
|
||||||
|
if not fm_open:
|
||||||
|
fm_open = True
|
||||||
|
continue
|
||||||
|
fm_close_idx = i
|
||||||
|
break
|
||||||
|
if fm_open and ln.startswith("last_updated:"):
|
||||||
|
lines[i] = f"last_updated: {today}"
|
||||||
|
bumped = True
|
||||||
|
if not fm_open:
|
||||||
|
print("index-append: warning: no frontmatter found, last_updated not bumped",
|
||||||
|
file=sys.stderr)
|
||||||
|
elif not bumped and fm_close_idx is not None:
|
||||||
|
lines.insert(fm_close_idx, f"last_updated: {today}")
|
||||||
|
print("index-append: last_updated key was missing — inserted", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def do_remove(lines, link, today):
|
||||||
|
"""Remove every entry line whose wikilink == link. Idempotent."""
|
||||||
|
bump_last_updated(lines, today)
|
||||||
|
kept = []
|
||||||
|
removed = 0
|
||||||
|
for ln in lines:
|
||||||
|
m = LINK_RE.match(ln)
|
||||||
|
if m and m.group(1) == link:
|
||||||
|
removed += 1
|
||||||
|
continue
|
||||||
|
kept.append(ln)
|
||||||
|
if removed:
|
||||||
|
print(f"index-append: removed [[{link}]] ({removed} line(s))")
|
||||||
|
else:
|
||||||
|
# Idempotent: the goal state (entry absent) already holds.
|
||||||
|
print(f"index-append: [[{link}]] not present, nothing to remove")
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
def do_append(lines, section, entry, today):
|
||||||
|
bump_last_updated(lines, today)
|
||||||
|
# Locate the target section [start, end)
|
||||||
|
start = None
|
||||||
|
for i, ln in enumerate(lines):
|
||||||
|
if HEADER_RE.match(ln) and ln[3:].startswith(section):
|
||||||
|
start = i
|
||||||
|
break
|
||||||
|
if start is None:
|
||||||
|
print(f"index-append: section '{section}' not found", file=sys.stderr)
|
||||||
|
return None
|
||||||
|
|
||||||
|
end = len(lines)
|
||||||
|
for i in range(start + 1, len(lines)):
|
||||||
|
if HEADER_RE.match(lines[i]):
|
||||||
|
end = i
|
||||||
|
break
|
||||||
|
|
||||||
|
body = lines[start + 1:end]
|
||||||
|
intro = [ln for ln in body if not ENTRY_RE.match(ln)]
|
||||||
|
entries = [ln for ln in body if ENTRY_RE.match(ln)]
|
||||||
|
|
||||||
|
new_m = LINK_RE.match(entry)
|
||||||
|
new_link = new_m.group(1) if new_m else None
|
||||||
|
|
||||||
|
if new_link is not None:
|
||||||
|
replaced = False
|
||||||
|
for idx, ln in enumerate(entries):
|
||||||
|
m = LINK_RE.match(ln)
|
||||||
|
if m and m.group(1) == new_link:
|
||||||
|
if ln == entry:
|
||||||
|
print("index-append: entry already present, skipping")
|
||||||
|
return lines
|
||||||
|
entries[idx] = entry
|
||||||
|
replaced = True
|
||||||
|
break
|
||||||
|
if not replaced:
|
||||||
|
entries.append(entry)
|
||||||
|
else:
|
||||||
|
if entry in entries:
|
||||||
|
print("index-append: entry already present, skipping")
|
||||||
|
return lines
|
||||||
|
entries.append(entry)
|
||||||
|
|
||||||
|
entries.sort(key=str.casefold)
|
||||||
|
while intro and intro[-1].strip() == "":
|
||||||
|
intro.pop()
|
||||||
|
new_section = intro + [""] + entries + [""]
|
||||||
|
print(f"index-append: added to {section}")
|
||||||
|
return lines[:start + 1] + new_section + lines[end:]
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--section", required=True,
|
ap.add_argument("--section", help="Section name (required with --entry)")
|
||||||
help="Section name, e.g. Sources / Entities / Concepts / Queries / Conflicts")
|
ap.add_argument("--entry", help="Full index line to insert")
|
||||||
ap.add_argument("--entry", required=True, help="Full index line to insert")
|
ap.add_argument("--remove", metavar="WIKILINK",
|
||||||
|
help="Remove the entry with this wikilink, e.g. sources/foo")
|
||||||
ap.add_argument("--file", default="wiki/index.md")
|
ap.add_argument("--file", default="wiki/index.md")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
if bool(args.remove) == bool(args.entry):
|
||||||
|
print("index-append: provide exactly one of --entry or --remove", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
if args.entry and not args.section:
|
||||||
|
print("index-append: --entry requires --section", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(args.file, encoding="utf-8") as fh:
|
with open(args.file, encoding="utf-8") as fh:
|
||||||
lines = fh.read().splitlines()
|
lines = fh.read().splitlines()
|
||||||
|
|
@ -33,90 +136,15 @@ def main() -> int:
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
today = datetime.date.today().isoformat()
|
today = datetime.date.today().isoformat()
|
||||||
|
if args.remove:
|
||||||
# 1. Bump last_updated inside the first frontmatter block
|
out = do_remove(lines, args.remove, today)
|
||||||
fm_open = False
|
|
||||||
fm_close_idx = None
|
|
||||||
bumped = False
|
|
||||||
for i, ln in enumerate(lines):
|
|
||||||
if ln.strip() == "---":
|
|
||||||
if not fm_open:
|
|
||||||
fm_open = True
|
|
||||||
continue
|
|
||||||
fm_close_idx = i # the closing ---
|
|
||||||
break
|
|
||||||
if fm_open and ln.startswith("last_updated:"):
|
|
||||||
lines[i] = f"last_updated: {today}"
|
|
||||||
bumped = True
|
|
||||||
|
|
||||||
if not fm_open:
|
|
||||||
print("index-append: warning: no frontmatter found, last_updated not bumped",
|
|
||||||
file=sys.stderr)
|
|
||||||
elif not bumped and fm_close_idx is not None:
|
|
||||||
# self-heal: frontmatter present but missing the key — insert it before the close
|
|
||||||
lines.insert(fm_close_idx, f"last_updated: {today}")
|
|
||||||
print("index-append: last_updated key was missing — inserted", file=sys.stderr)
|
|
||||||
|
|
||||||
# 2. Locate the target section [start, end)
|
|
||||||
start = None
|
|
||||||
for i, ln in enumerate(lines):
|
|
||||||
if HEADER_RE.match(ln) and ln[3:].startswith(args.section):
|
|
||||||
start = i
|
|
||||||
break
|
|
||||||
if start is None:
|
|
||||||
print(f"index-append: section '{args.section}' not found in {args.file}",
|
|
||||||
file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
end = len(lines)
|
|
||||||
for i in range(start + 1, len(lines)):
|
|
||||||
if HEADER_RE.match(lines[i]):
|
|
||||||
end = i
|
|
||||||
break
|
|
||||||
|
|
||||||
# 3. Split the section body into intro (non-entry) and entries
|
|
||||||
body = lines[start + 1:end]
|
|
||||||
intro = [ln for ln in body if not ENTRY_RE.match(ln)]
|
|
||||||
entries = [ln for ln in body if ENTRY_RE.match(ln)]
|
|
||||||
|
|
||||||
# Deduplicate by wikilink PATH, not by exact line: a re-ingest with a changed
|
|
||||||
# summary/maturity should UPDATE the existing entry, not add a duplicate line.
|
|
||||||
new_m = LINK_RE.match(args.entry)
|
|
||||||
new_link = new_m.group(1) if new_m else None
|
|
||||||
|
|
||||||
if new_link is not None:
|
|
||||||
replaced = False
|
|
||||||
for idx, ln in enumerate(entries):
|
|
||||||
m = LINK_RE.match(ln)
|
|
||||||
if m and m.group(1) == new_link:
|
|
||||||
if ln == args.entry:
|
|
||||||
print("index-append: entry already present, skipping")
|
|
||||||
return 0
|
|
||||||
entries[idx] = args.entry # same page, refreshed text
|
|
||||||
replaced = True
|
|
||||||
break
|
|
||||||
if not replaced:
|
|
||||||
entries.append(args.entry)
|
|
||||||
else:
|
else:
|
||||||
# No parseable wikilink — fall back to exact-line dedup.
|
out = do_append(lines, args.section, args.entry, today)
|
||||||
if args.entry in entries:
|
if out is None:
|
||||||
print("index-append: entry already present, skipping")
|
return 1
|
||||||
return 0
|
|
||||||
entries.append(args.entry)
|
|
||||||
|
|
||||||
entries.sort(key=str.casefold)
|
|
||||||
|
|
||||||
# Normalise intro: drop trailing blanks, keep header + comment(s)
|
|
||||||
while intro and intro[-1].strip() == "":
|
|
||||||
intro.pop()
|
|
||||||
|
|
||||||
new_section = intro + [""] + entries + [""]
|
|
||||||
lines = lines[:start + 1] + new_section + lines[end:]
|
|
||||||
|
|
||||||
with open(args.file, "w", encoding="utf-8") as fh:
|
with open(args.file, "w", encoding="utf-8") as fh:
|
||||||
fh.write("\n".join(lines) + "\n")
|
fh.write("\n".join(out) + "\n")
|
||||||
|
|
||||||
print(f"index-append: added to {args.section}")
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,11 @@ set -euo pipefail
|
||||||
: "${FORGEJO_USER:?missing FORGEJO_USER}"
|
: "${FORGEJO_USER:?missing FORGEJO_USER}"
|
||||||
: "${FORGEJO_TOKEN:?missing FORGEJO_TOKEN}"
|
: "${FORGEJO_TOKEN:?missing FORGEJO_TOKEN}"
|
||||||
|
|
||||||
slug="" title="" body_file="" base="main" label=""
|
slug="" title="" body_file="" base="main" label="" branch=""
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--slug) slug="$2"; shift 2 ;;
|
--slug) slug="$2"; shift 2 ;;
|
||||||
|
--branch) branch="$2"; shift 2 ;;
|
||||||
--title) title="$2"; shift 2 ;;
|
--title) title="$2"; shift 2 ;;
|
||||||
--body-file) body_file="$2"; shift 2 ;;
|
--body-file) body_file="$2"; shift 2 ;;
|
||||||
--base) base="$2"; shift 2 ;;
|
--base) base="$2"; shift 2 ;;
|
||||||
|
|
@ -28,16 +29,23 @@ while [[ $# -gt 0 ]]; do
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
: "${slug:?--slug required}"
|
|
||||||
: "${title:?--title required}"
|
: "${title:?--title required}"
|
||||||
: "${body_file:?--body-file required}"
|
: "${body_file:?--body-file required}"
|
||||||
[[ -f "$body_file" ]] || { echo "open-pr: body file not found: $body_file" >&2; exit 1; }
|
[[ -f "$body_file" ]] || { echo "open-pr: body file not found: $body_file" >&2; exit 1; }
|
||||||
|
|
||||||
branch="feat/ai-ingest-${slug}"
|
# --branch overrides the default; otherwise derive the ingest branch from --slug.
|
||||||
|
# (run-prune passes its own chore/prune-orphans-* branch; run-ingest passes --slug.)
|
||||||
|
if [[ -z "$branch" ]]; then
|
||||||
|
: "${slug:?--slug or --branch required}"
|
||||||
|
branch="feat/ai-ingest-${slug}"
|
||||||
|
fi
|
||||||
repo="$(basename -s .git "$(git config --get remote.origin.url)")"
|
repo="$(basename -s .git "$(git config --get remote.origin.url)")"
|
||||||
|
|
||||||
# 1. Branch + commit + push (AGENTS.md rule 5: never commit to main)
|
# 1. Branch + commit + push (AGENTS.md rule 5: never commit to main)
|
||||||
git switch -c "$branch" 2>/dev/null || git switch "$branch"
|
# Rolling PR: -C force-resets the branch label to the current base (we are on it after
|
||||||
|
# clean_start) and CARRIES the freshly-written wiki/ changes, so a re-ingest of the same
|
||||||
|
# source rebuilds the branch cleanly instead of hitting a dirty-switch refusal.
|
||||||
|
git switch -C "$branch"
|
||||||
git add wiki/
|
git add wiki/
|
||||||
# Scope BOTH the emptiness check and the commit to wiki/ — never commit anything that
|
# Scope BOTH the emptiness check and the commit to wiki/ — never commit anything that
|
||||||
# happened to be staged outside wiki/ (a stray hook, an aborted prior run, etc.).
|
# happened to be staged outside wiki/ (a stray hook, an aborted prior run, etc.).
|
||||||
|
|
@ -46,7 +54,10 @@ if git diff --cached --quiet -- wiki/; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
git commit -m "$title" -- wiki/
|
git commit -m "$title" -- wiki/
|
||||||
git push -u origin "$branch"
|
# Try a normal push (new branch / fast-forward). If the branch was rebuilt from base and
|
||||||
|
# diverged, force-with-lease updates the open PR in place — the lease refuses to clobber if
|
||||||
|
# origin moved unexpectedly since our fetch, so concurrent work is never lost.
|
||||||
|
git push -u origin "$branch" 2>/dev/null || git push -u --force-with-lease origin "$branch"
|
||||||
|
|
||||||
# DRY_RUN: local git work done; skip the Forgejo API (offline tests).
|
# DRY_RUN: local git work done; skip the Forgejo API (offline tests).
|
||||||
if [[ -n "${DRY_RUN:-}" ]]; then
|
if [[ -n "${DRY_RUN:-}" ]]; then
|
||||||
|
|
|
||||||
96
skills/ingest/scripts/run-prune.sh
Executable file
96
skills/ingest/scripts/run-prune.sh
Executable file
|
|
@ -0,0 +1,96 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================================
|
||||||
|
# skills/ingest/scripts/run-prune.sh
|
||||||
|
# Symmetric companion to run-ingest: prune source pages whose raw source no
|
||||||
|
# longer exists. RE-DERIVES the orphan set itself (mirrors orphan-wiki.sh) — it
|
||||||
|
# never trusts a list handed in by n8n, so there is no "detected-vs-pruned"
|
||||||
|
# race. Removes ONLY the pages it derived plus their index entries, commits
|
||||||
|
# ONLY wiki/ on chore/prune-orphans-<date>, and opens a GATED removal PR (the
|
||||||
|
# operator approves the deletion; principle 2). Never deletes of its own accord.
|
||||||
|
#
|
||||||
|
# Runs OUTSIDE the model, on vm101, cwd = genome checkout. The wrapper (`pi
|
||||||
|
# prune`) has already taken the per-genome lock and done clean_start, exactly
|
||||||
|
# like `pi ingest` — so this script does neither.
|
||||||
|
#
|
||||||
|
# run-prune.sh <genome>
|
||||||
|
#
|
||||||
|
# Emits a single JSON result line on stdout for n8n to parse.
|
||||||
|
# =============================================================================
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
genome="${1:?usage: run-prune.sh <genome>}"
|
||||||
|
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
jq -nc --arg stage "$1" --arg reason "$2" '{status:"error", stage:$stage, reason:$reason}'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
command -v jq >/dev/null 2>&1 || { echo '{"status":"error","reason":"jq missing"}'; exit 1; }
|
||||||
|
command -v python3 >/dev/null 2>&1 || fail "deps" "python3 missing (needed by index-append.py)"
|
||||||
|
|
||||||
|
# --- re-derive orphans (same rule as orphan-wiki.sh; computed fresh, here, now) ---
|
||||||
|
# A wiki/sources/*.md page is orphaned when its frontmatter source_path points at
|
||||||
|
# a raw file that no longer exists. Legacy pages without source_path are ignored.
|
||||||
|
declare -a ORPH=()
|
||||||
|
for page in wiki/sources/*.md; do
|
||||||
|
[[ -e "$page" ]] || continue
|
||||||
|
sp="$(sed -n 's/^source_path:[[:space:]]*//p' "$page" | tr -d '\r' | head -n1)"
|
||||||
|
[[ -n "$sp" ]] || continue
|
||||||
|
[[ -f "$sp" ]] || ORPH+=("$page")
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#ORPH[@]} -eq 0 ]]; then
|
||||||
|
jq -nc '{status:"ok", count:0, pruned:[], detail:"no orphans"}'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- remove each orphan page + its index entry (anti-traversal, wiki/-only) ---
|
||||||
|
declare -a PRUNED=()
|
||||||
|
for page in "${ORPH[@]}"; do
|
||||||
|
case "$page" in
|
||||||
|
wiki/*) : ;;
|
||||||
|
*) fail "prune" "refusing to remove outside wiki/: ${page}" ;;
|
||||||
|
esac
|
||||||
|
case "$page" in *..*) fail "prune" "path traversal in page: ${page}" ;; esac
|
||||||
|
[[ -f "$page" ]] || continue
|
||||||
|
rm -f "$page"
|
||||||
|
link="${page#wiki/}"; link="${link%.md}" # e.g. sources/foo
|
||||||
|
python3 "${SCRIPTS}/index-append.py" --remove "$link" \
|
||||||
|
|| fail "index" "index-append --remove failed for ${link}"
|
||||||
|
PRUNED+=("$link")
|
||||||
|
done
|
||||||
|
|
||||||
|
# --- assemble the PR body ---
|
||||||
|
date_tag="$(date +%F)"
|
||||||
|
body="$(mktemp)"
|
||||||
|
trap 'rm -f "$body"' EXIT
|
||||||
|
{
|
||||||
|
echo "## Prune orphaned sources"
|
||||||
|
echo ""
|
||||||
|
echo "These source pages reference a \`source_path\` whose raw file no longer exists"
|
||||||
|
echo "in \`raw/\`. Removing them keeps the wiki in sync with git (the source of truth)."
|
||||||
|
echo ""
|
||||||
|
echo "| Removed page |"
|
||||||
|
echo "|--------------|"
|
||||||
|
for l in "${PRUNED[@]}"; do echo "| \`wiki/${l}.md\` |"; done
|
||||||
|
} > "$body"
|
||||||
|
|
||||||
|
# --- open the GATED removal PR on a chore/ branch (open-pr --branch override) ---
|
||||||
|
branch="chore/prune-orphans-${date_tag}"
|
||||||
|
pr_out="$( bash "${SCRIPTS}/open-pr.sh" \
|
||||||
|
--branch "$branch" \
|
||||||
|
--title "chore: prune ${#PRUNED[@]} orphaned source(s)" \
|
||||||
|
--body-file "$body" --base "${INGEST_BASE:-main}" 2>&1 )" && pr_rc=0 || pr_rc=$?
|
||||||
|
pr_url="$(printf '%s\n' "$pr_out" | sed -n 's/^PR opened: //p' | head -n1)"
|
||||||
|
|
||||||
|
# --- result line for n8n ---
|
||||||
|
jq -nc \
|
||||||
|
--arg status "$([[ $pr_rc -eq 0 ]] && echo ok || echo pr_failed)" \
|
||||||
|
--argjson count "${#PRUNED[@]}" \
|
||||||
|
--arg pr_url "$pr_url" \
|
||||||
|
--arg detail "$pr_out" \
|
||||||
|
--argjson pruned "$(printf '%s\n' "${PRUNED[@]}" | jq -R . | jq -s .)" \
|
||||||
|
'{status:$status, count:$count, pr_url:$pr_url, pruned:$pruned, detail:$detail}'
|
||||||
|
|
||||||
|
[[ $pr_rc -eq 0 ]] || exit 1
|
||||||
44
tests/index-remove.bats
Normal file
44
tests/index-remove.bats
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# tests/index-remove.bats — index-append.py --remove mode.
|
||||||
|
setup() {
|
||||||
|
load 'helpers'
|
||||||
|
export GENOMES_ROOT="${BATS_TEST_TMPDIR}"
|
||||||
|
g_src="$(make_fixture_genome)"; export g="$g_src"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "index --remove: deletes the matching entry, keeps the others" {
|
||||||
|
cd "$g"
|
||||||
|
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — A. `maturity: draft`'
|
||||||
|
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/b]] — B. `maturity: draft`'
|
||||||
|
grep -q 'sources/a' wiki/index.md
|
||||||
|
grep -q 'sources/b' wiki/index.md
|
||||||
|
|
||||||
|
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/a'
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
! grep -q '\[\[sources/a\]\]' wiki/index.md
|
||||||
|
grep -q 'sources/b' wiki/index.md
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "index --remove: idempotent when the entry is absent" {
|
||||||
|
cd "$g"
|
||||||
|
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/does-not-exist'
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *'nothing to remove'* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "index --remove: bumps last_updated" {
|
||||||
|
cd "$g"
|
||||||
|
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — A. `maturity: draft`'
|
||||||
|
# set last_updated to an old date, then remove and check it moved
|
||||||
|
sed -i 's/^last_updated:.*/last_updated: 2000-01-01/' wiki/index.md
|
||||||
|
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/a'
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
! grep -q '2000-01-01' wiki/index.md
|
||||||
|
grep -q "last_updated: $(date +%F)" wiki/index.md
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "index --remove: rejects passing both --entry and --remove" {
|
||||||
|
cd "$g"
|
||||||
|
run python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — x' --remove 'sources/a'
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
}
|
||||||
48
tests/open-pr-rolling.bats
Normal file
48
tests/open-pr-rolling.bats
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# open-pr-rolling.bats — a re-ingest of the same slug updates the OPEN PR's branch
|
||||||
|
# (force-with-lease) instead of failing. Uses the local bare remote from make_fixture_genome.
|
||||||
|
load helpers
|
||||||
|
setup_file() { :; }
|
||||||
|
|
||||||
|
@test "open-pr: re-ingest of the same slug rolls the branch forward (force-with-lease)" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
G="$(make_fixture_genome)"; cd "$G"
|
||||||
|
export FORGEJO_URL="http://forgejo.local" FORGEJO_USER=u FORGEJO_TOKEN=t DRY_RUN=1
|
||||||
|
body="$(mktemp)"; echo body > "$body"
|
||||||
|
|
||||||
|
# first ingest of slug x (v1)
|
||||||
|
mkdir -p wiki/sources; printf 'v1\n' > wiki/sources/x.md
|
||||||
|
run bash "$SKILL_SCRIPTS/open-pr.sh" --slug x --title "feat: ingest x" --body-file "$body" --base main
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
git rev-parse --verify feat/ai-ingest-x
|
||||||
|
first="$(git rev-parse feat/ai-ingest-x)"
|
||||||
|
|
||||||
|
# simulate clean_start back to base, then an edited re-ingest (v2)
|
||||||
|
git switch -q main; git reset -q --hard origin/main; git clean -q -fd
|
||||||
|
printf 'v2-edited\n' > wiki/sources/x.md
|
||||||
|
run bash "$SKILL_SCRIPTS/open-pr.sh" --slug x --title "feat: ingest x" --body-file "$body" --base main
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
second="$(git rev-parse feat/ai-ingest-x)"
|
||||||
|
|
||||||
|
# the branch was REBUILT from base (diverged), not appended: second is not a descendant of first
|
||||||
|
run git merge-base --is-ancestor "$first" "$second"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
|
||||||
|
# origin received the v2 content (force-with-lease pushed the rebuilt branch)
|
||||||
|
git fetch -q origin
|
||||||
|
run git show "origin/feat/ai-ingest-x:wiki/sources/x.md"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"v2-edited"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "open-pr: prune branch override still works after the rolling change" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
G="$(make_fixture_genome)"; cd "$G"
|
||||||
|
export FORGEJO_URL="http://forgejo.local" FORGEJO_USER=u FORGEJO_TOKEN=t DRY_RUN=1
|
||||||
|
body="$(mktemp)"; echo body > "$body"
|
||||||
|
mkdir -p wiki/sources; printf 'p\n' > wiki/sources/p.md
|
||||||
|
run bash "$SKILL_SCRIPTS/open-pr.sh" --branch "chore/prune-orphans-2026-06-30" \
|
||||||
|
--title "chore: prune 1 orphaned source(s)" --body-file "$body" --base main
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
git rev-parse --verify "chore/prune-orphans-2026-06-30"
|
||||||
|
}
|
||||||
75
tests/raw-commit-quiet.bats
Normal file
75
tests/raw-commit-quiet.bats
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# raw-commit-quiet.bats — quiet-window behaviour of genome-raw-commit.sh.
|
||||||
|
# No Syncthing (no API key -> default author); pushes to a local bare repo via GENOME_PUSH_URL.
|
||||||
|
setup() {
|
||||||
|
SCRIPT="${BATS_TEST_DIRNAME}/../deploy/nexus/genome-raw-commit.sh"
|
||||||
|
export HOME="${BATS_TEST_TMPDIR}/home"; mkdir -p "$HOME/.config"
|
||||||
|
root="${BATS_TEST_TMPDIR}/vaults"; mkdir -p "$root"
|
||||||
|
bare="${BATS_TEST_TMPDIR}/origin.git"; git init -q --bare "$bare"
|
||||||
|
cat > "$HOME/.config/knowledge-genome.env" <<EOF
|
||||||
|
GENOME_VAULTS_ROOT=$root
|
||||||
|
GENOME_BASE=main
|
||||||
|
FORGEJO_USER=n8n-bot
|
||||||
|
FORGEJO_HOST=127.0.0.1:3001
|
||||||
|
FORGEJO_OWNER=Keru
|
||||||
|
COMMITTER_NAME=n8n-bot
|
||||||
|
COMMITTER_EMAIL=n8n-bot@homelab
|
||||||
|
DEFAULT_AUTHOR_NAME=Tester
|
||||||
|
DEFAULT_AUTHOR_EMAIL=tester@local
|
||||||
|
EOF
|
||||||
|
export g="genome-test"; export vault="$root/$g"
|
||||||
|
git clone -q "$bare" "$vault" 2>/dev/null || mkdir -p "$vault"
|
||||||
|
( cd "$vault"
|
||||||
|
git init -q 2>/dev/null || true
|
||||||
|
git config user.name n8n-bot; git config user.email n8n-bot@homelab; git config commit.gpgsign false
|
||||||
|
git checkout -q -b main 2>/dev/null || git switch -q main
|
||||||
|
mkdir -p raw/articles; echo seed > raw/articles/.gitkeep
|
||||||
|
git add -A; git commit -q -m init
|
||||||
|
git remote add origin "$bare" 2>/dev/null || git remote set-url origin "$bare"
|
||||||
|
git push -q -u origin main )
|
||||||
|
export GENOME_PUSH_URL="$bare" # test seam -> push to the local bare repo
|
||||||
|
}
|
||||||
|
files() { ( cd "$vault" && git ls-files raw/ ) > "${BATS_TEST_TMPDIR}/f.txt"; }
|
||||||
|
|
||||||
|
@test "raw-commit: holds a freshly-written raw, commits it once it settles" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
echo "still typing" > "$vault/raw/articles/hot.md" # fresh -> hot
|
||||||
|
echo "finished" > "$vault/raw/articles/stable.md"
|
||||||
|
touch -d "10 minutes ago" "$vault/raw/articles/stable.md" # settled
|
||||||
|
|
||||||
|
run bash "$SCRIPT" "$g"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
echo "$output" | jq -e '.status=="ok"'
|
||||||
|
files
|
||||||
|
grep -q 'raw/articles/stable.md' "${BATS_TEST_TMPDIR}/f.txt" # committed
|
||||||
|
! grep -q 'raw/articles/hot.md' "${BATS_TEST_TMPDIR}/f.txt" # held back
|
||||||
|
|
||||||
|
touch -d "10 minutes ago" "$vault/raw/articles/hot.md" # now it settles
|
||||||
|
run bash "$SCRIPT" "$g"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
files
|
||||||
|
grep -q 'raw/articles/hot.md' "${BATS_TEST_TMPDIR}/f.txt" # now committed
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "raw-commit: noop with held count while everything is still settling" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
echo "typing" > "$vault/raw/articles/wip.md" # fresh -> hot
|
||||||
|
run bash "$SCRIPT" "$g"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
echo "$output" | jq -e '.status=="noop"'
|
||||||
|
echo "$output" | jq -e '.held==1'
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "raw-commit: a deletion is committed immediately (not subject to the quiet window)" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
# commit a settled file first
|
||||||
|
echo done > "$vault/raw/articles/old.md"; touch -d "10 minutes ago" "$vault/raw/articles/old.md"
|
||||||
|
run bash "$SCRIPT" "$g"; [ "$status" -eq 0 ]
|
||||||
|
files; grep -q 'raw/articles/old.md' "${BATS_TEST_TMPDIR}/f.txt"
|
||||||
|
# now delete it -> should commit the removal even though "just changed"
|
||||||
|
rm "$vault/raw/articles/old.md"
|
||||||
|
run bash "$SCRIPT" "$g"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
echo "$output" | jq -e '.status=="ok"'
|
||||||
|
files; ! grep -q 'raw/articles/old.md' "${BATS_TEST_TMPDIR}/f.txt"
|
||||||
|
}
|
||||||
68
tests/run-prune.bats
Normal file
68
tests/run-prune.bats
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
#!/usr/bin/env bats
|
||||||
|
# tests/run-prune.bats — prune orphaned sources (no LLM, no network; DRY_RUN).
|
||||||
|
setup() {
|
||||||
|
load 'helpers'
|
||||||
|
export PRUNE="${SKILL_SCRIPTS}/run-prune.sh"
|
||||||
|
export GENOMES_ROOT="${BATS_TEST_TMPDIR}"
|
||||||
|
export INGEST_BASE="main"
|
||||||
|
export KG_LIB_DIR="${LIB_DIR}"
|
||||||
|
export FORGEJO_URL="http://forgejo.local" FORGEJO_USER="u" FORGEJO_TOKEN="t"
|
||||||
|
export DRY_RUN=1
|
||||||
|
g_src="$(make_fixture_genome)"; export g_name="fixture-genome"
|
||||||
|
mv "$g_src" "${GENOMES_ROOT}/${g_name}"; export g="${GENOMES_ROOT}/${g_name}"
|
||||||
|
( cd "$g" && rm -f raw/articles/test.md && git add -A && git commit -q -m clear && git push -q )
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run-prune: removes only the orphaned source + its index entry, opens a dry PR" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
cd "$g"
|
||||||
|
# kept: raw exists. orphan: raw missing.
|
||||||
|
echo content > raw/articles/kept.md
|
||||||
|
h="$(sha256sum raw/articles/kept.md | cut -d' ' -f1)"
|
||||||
|
printf -- '---\nsource_path: raw/articles/kept.md\nsource_sha256: %s\n---\nbody\n' "$h" > wiki/sources/kept.md
|
||||||
|
printf -- '---\nsource_path: raw/articles/gone.md\nsource_sha256: abc\n---\nbody\n' > wiki/sources/orphan.md
|
||||||
|
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/kept]] — kept. `maturity: draft`'
|
||||||
|
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/orphan]] — orphan. `maturity: draft`'
|
||||||
|
git add -A && git commit -q -m setup && git push -q
|
||||||
|
|
||||||
|
run bash "$PRUNE" "$g_name"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *'"status":"ok"'* ]]
|
||||||
|
[[ "$output" == *'"count":1'* ]]
|
||||||
|
|
||||||
|
# only the orphan page is gone
|
||||||
|
[ ! -f wiki/sources/orphan.md ]
|
||||||
|
[ -f wiki/sources/kept.md ]
|
||||||
|
# index reflects the removal
|
||||||
|
! grep -q 'sources/orphan' wiki/index.md
|
||||||
|
grep -q 'sources/kept' wiki/index.md
|
||||||
|
# committed on a chore/ branch (NOT feat/ai-ingest-*)
|
||||||
|
git rev-parse --verify "chore/prune-orphans-$(date +%F)"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run-prune: no orphans -> count 0 and no PR/branch" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
cd "$g"
|
||||||
|
echo content > raw/articles/kept.md
|
||||||
|
h="$(sha256sum raw/articles/kept.md | cut -d' ' -f1)"
|
||||||
|
printf -- '---\nsource_path: raw/articles/kept.md\nsource_sha256: %s\n---\nbody\n' "$h" > wiki/sources/kept.md
|
||||||
|
git add -A && git commit -q -m setup && git push -q
|
||||||
|
|
||||||
|
run bash "$PRUNE" "$g_name"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *'"count":0'* ]]
|
||||||
|
run git rev-parse --verify "chore/prune-orphans-$(date +%F)"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "run-prune: refuses when an orphan path would escape wiki/ (defense in depth)" {
|
||||||
|
command -v jq >/dev/null 2>&1 || skip "jq not installed"
|
||||||
|
cd "$g"
|
||||||
|
# legacy page without source_path is ignored; a page with a missing raw is the orphan.
|
||||||
|
printf -- '---\nsource_path: raw/articles/gone.md\nsource_sha256: abc\n---\nbody\n' > wiki/sources/orphan.md
|
||||||
|
git add -A && git commit -q -m setup && git push -q
|
||||||
|
run bash "$PRUNE" "$g_name"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *'"count":1'* ]]
|
||||||
|
[ ! -f wiki/sources/orphan.md ]
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue