Compare commits

...

16 commits

Author SHA1 Message Date
601277fce4 Merge branch 'release/1.11.1' into main 2026-07-02 10:15:45 +02:00
fd5c07043c Update version 2026-07-02 10:15:32 +02:00
c4aba8507c feat(n8n): add workflow for genome content pruning on raw/ file deletion 2026-07-01 19:53:57 +02:00
1c6d7a4ecd feat(n8n): add 'pi prune' command to n8n-pi-wrap 2026-07-01 19:37:14 +02:00
066db00e89 test(ingest): add tests for run-prune.sh 2026-07-01 19:37:14 +02:00
c0659d5ce9 feat(ingest): introduce run-prune.sh for orphaned source removal 2026-07-01 19:37:14 +02:00
101eef98aa feat(infra): enhance open-pr.sh for rolling PRs and custom branches 2026-07-01 19:37:14 +02:00
8082bc3003 test(ingest): add tests for index-append.py --remove 2026-07-01 19:37:14 +02:00
990118de71 feat(ingest): allow index-append.py to remove entries 2026-07-01 19:37:14 +02:00
95b3866549 feat(ingest-semantic.py): Add pre-flight context window check 2026-07-01 18:24:25 +02:00
111ffd266a refactor(genome-ingest): Use run-one-ingest and improve filtering 2026-07-01 18:24:25 +02:00
88aa6a0798 refactor: Remove old raw-commit workflow 2026-07-01 18:24:25 +02:00
79c4f6dde2 feat: Implement PR review directive workflow 2026-07-01 18:24:25 +02:00
047330b384 feat: Introduce run-one-ingest sub-workflow 2026-07-01 18:24:25 +02:00
c8b45d537c feat: Add global n8n error handling workflow 2026-07-01 18:24:25 +02:00
e57b811956 Merge branch 'release/1.11.0' into develop 2026-06-30 10:33:14 +02:00
14 changed files with 2054 additions and 494 deletions

View file

@ -1,5 +1,5 @@
# =============================================================================
# Knowledge Genome - Makefile v. 1.11.0
# Knowledge Genome - Makefile v. 1.11.1
# Orchestrates the setup and management of the knowledge base.
# =============================================================================

View file

@ -0,0 +1,773 @@
{
"name": "Genome: PR review",
"nodes": [
{
"parameters": {
"httpMethod": "POST",
"path": "forgejo-pr-review-forgejo-pr-review-23319ab8687b16f10e0f278fb920c112",
"options": {}
},
"id": "edf8e431-3637-477d-83bd-1f077843f740",
"name": "Webhook PR Review",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2.1,
"position": [
-144,
304
],
"webhookId": "61ff3a5baa304571"
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// THE only parser of the review side: parse the directive, VALIDATE, prepare the rework payload.\n// Security: only allow-listed maintainers may drive the gate; destructive directives require a\n// feat/ai-ingest-* branch on the expected base; raw_source is recovered from a machine-readable\n// marker that run-ingest.sh writes into the PR body.\nconst ALLOWED_SENDERS = ['Keru']; // <-- maintainers allowed to issue directives\nconst BASE = 'develop';\nconst j = $json.body || $json;\nconst review = j.review || null, comment = j.comment || null;\nconst pr = j.pull_request || j.issue || null;\nconst body = ((review && review.content) || (comment && comment.body) || '').toString();\nconst sender = (j.sender && j.sender.login) || 'unknown';\n\nconst m = body.match(/^\\s*(REWORK|RESTART|REVERT\\s+\\d+|SPLIT|REJECT|MERGE)\\s*:?/i);\nif (!m) return { directive: 'NONE' };\nconst headTok = m[1].toUpperCase().replace(/\\s+/g, ' ');\nlet directive = headTok.startsWith('REVERT') ? 'REVERT' : headTok;\nconst feedback = body.slice(m[0].length).trim() || '(nessun dettaglio fornito)';\n\nconst prNumber = (pr && pr.number) || null;\nconst branch = (pr && pr.head && pr.head.ref) || null;\nconst base = (pr && pr.base && pr.base.ref) || null;\nconst repo = (pr && pr.base && pr.base.repo && pr.base.repo.name) || (j.repository && j.repository.name) || null;\nconst owner = (pr && pr.base && pr.base.repo && pr.base.repo.owner && pr.base.repo.owner.login)\n || (j.repository && j.repository.owner && j.repository.owner.login) || null;\nconst prBody = (pr && pr.body) || (j.issue && j.issue.body) || '';\nconst rawMatch = prBody.match(/<!--\\s*kg:raw=([^\\s]+)\\s*-->/);\nconst raw = rawMatch ? rawMatch[1] : null;\n\nif (directive === 'REVERT') return { directive: 'NONE', note: 'REVERT reserved for Step 7' };\nif (!ALLOWED_SENDERS.includes(sender))\n return { directive: 'UNAUTHORIZED', attempted: directive, sender, prNumber, owner, repo };\n\nconst okGenome = !!repo && /^[a-z0-9][a-z0-9-]{0,63}$/.test(repo);\nconst okPr = !!prNumber && /^[0-9]+$/.test(String(prNumber));\nconst okBranch = !!branch && /^feat\\/ai-ingest-[a-z0-9-]+$/.test(branch);\nconst okBase = base === BASE;\nconst okRaw = (directive === 'MERGE') ? true\n : (!!raw && raw.startsWith('raw/') && !raw.includes('..') && /^[A-Za-z0-9._\\/-]+$/.test(raw));\nif (!okGenome || !okPr || !okBase || (directive !== 'MERGE' && !okBranch) || !okRaw)\n return { directive: 'INVALID', attempted: directive, prNumber, owner, repo,\n why: { okGenome, okPr, okBranch, okBase, okRaw } };\n\nconst feedback_b64 = Buffer.from(feedback, 'utf8').toString('base64');\nreturn { directive, prNumber, branch, base, repo, owner, sender, raw, feedback, feedback_b64 };"
},
"id": "39977823-cbc1-45bb-b479-a57052b482e9",
"name": "Parse & validate",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
80,
304
]
},
{
"parameters": {
"rules": {
"values": [
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "MERGE",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "4960f0868bc54687"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "REWORK",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "34002fdd92834d38"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "RESTART",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "d412a74e32ac4f0c"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "SPLIT",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "c0810b33fa474ca0"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "REJECT",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "531039e699c44cea"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "UNAUTHORIZED",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "cfbd691d2e9a4c2a"
}
],
"combinator": "and"
}
},
{
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"leftValue": "={{ $json.directive }}",
"rightValue": "INVALID",
"operator": {
"type": "string",
"operation": "equals"
},
"id": "251f5b7beea6424a"
}
],
"combinator": "and"
}
}
]
},
"options": {
"fallbackOutput": "none"
}
},
"id": "bc8aff39-a6bf-4e7c-8069-505d5855fb62",
"name": "Switch",
"type": "n8n-nodes-base.switch",
"typeVersion": 3.4,
"position": [
320,
304
]
},
{
"parameters": {
"method": "POST",
"url": "=https://git.keruhomelab.com/api/v1/repos/{{ $('Parse & validate').first().json.owner }}/{{ $('Parse & validate').first().json.repo }}/pulls/{{ $('Parse & validate').first().json.prNumber }}/merge",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "={\n \"Do\": \"merge\"\n}",
"options": {
"timeout": 15000
}
},
"id": "2cea722f-42f1-475e-8060-7bac7cf4d245",
"name": "Forgejo Merge PR",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
560,
64
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
}
}
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "cc369b5fc3d246a4",
"leftValue": "={{ $('Parse & validate').first().json.branch }}",
"rightValue": "feat/ai-ingest-",
"operator": {
"type": "string",
"operation": "startsWith"
}
}
],
"combinator": "and"
},
"options": {}
},
"id": "aff153d5-48c8-4a31-bd37-5bce49e60fa9",
"name": "Guardia feat/",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
560,
288
]
},
{
"parameters": {
"method": "PATCH",
"url": "=https://git.keruhomelab.com/api/v1/repos/{{ $('Parse & validate').first().json.owner }}/{{ $('Parse & validate').first().json.repo }}/pulls/{{ $('Parse & validate').first().json.prNumber }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "={\n \"state\": \"closed\"\n}",
"options": {
"timeout": 15000
}
},
"id": "1745f043-9dc3-44e2-8654-4cc88114d636",
"name": "Forgejo Close PR",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
784,
256
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
}
}
},
{
"parameters": {
"method": "DELETE",
"url": "=https://git.keruhomelab.com/api/v1/repos/{{ $('Parse & validate').first().json.owner }}/{{ $('Parse & validate').first().json.repo }}/branches/{{ encodeURIComponent($('Parse & validate').first().json.branch) }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"timeout": 15000
}
},
"id": "de59c610-c671-4a48-bca4-61ba9988bc65",
"name": "Forgejo Delete Branch",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
1008,
256
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
}
}
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"id": "55cf6c2a6c7d4d79",
"leftValue": "={{ $('Parse & validate').first().json.directive }}",
"rightValue": "REJECT",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"combinator": "and"
},
"options": {}
},
"id": "5c149f65-1ce2-4a39-9b86-aa05a993735c",
"name": "E' REJECT?",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
1232,
256
]
},
{
"parameters": {
"workflowId": {
"__rl": true,
"value": "zbtRXWsLt56nEIfz",
"mode": "list",
"cachedResultUrl": "/workflow/zbtRXWsLt56nEIfz",
"cachedResultName": "Power Manager"
},
"workflowInputs": {
"mappingMode": "defineBelow",
"value": {
"mode": "ensure-on"
},
"matchingColumns": [
"mode"
],
"schema": [
{
"id": "mode",
"displayName": "mode",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
}
],
"attemptToConvertTypes": false,
"convertFieldsToString": true
},
"options": {}
},
"id": "24900f7e-959e-4398-8630-721a38443aa4",
"name": "Power Manager - ensure-on",
"type": "n8n-nodes-base.executeWorkflow",
"typeVersion": 1.3,
"position": [
1232,
128
]
},
{
"parameters": {
"workflowId": {
"__rl": true,
"value": "VIi2ovb5gJxNJLbg",
"mode": "list",
"cachedResultUrl": "/workflow/VIi2ovb5gJxNJLbg",
"cachedResultName": "Genome: run-one-ingest"
},
"workflowInputs": {
"mappingMode": "defineBelow",
"value": {
"genome": "={{ $('Parse & validate').first().json.repo }}",
"raw": "={{ $('Parse & validate').first().json.raw }}",
"mode": "rework",
"feedback_b64": "={{ $('Parse & validate').first().json.feedback_b64 }}",
"reason": "={{ $('Parse & validate').first().json.directive }}",
"prevPr": "={{ $('Parse & validate').first().json.prNumber }}"
},
"matchingColumns": [],
"schema": [
{
"id": "genome",
"displayName": "genome",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "raw",
"displayName": "raw",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "mode",
"displayName": "mode",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "feedback_b64",
"displayName": "feedback_b64",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "reason",
"displayName": "reason",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "prevPr",
"displayName": "prevPr",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
}
],
"attemptToConvertTypes": false,
"convertFieldsToString": true
},
"options": {
"waitForSubWorkflow": false
}
},
"id": "16774a78-b4eb-491f-9508-040aa3d4dc12",
"name": "Run one ingest (rework)",
"type": "n8n-nodes-base.executeWorkflow",
"typeVersion": 1.3,
"position": [
1440,
128
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// merged (MERGE) / closed (REJECT). The HTTP node replaced $json with the API response, so read\n// context from the parser (single review -> .first() is correct and pairedItem-proof).\nconst p = $('Parse & validate').first().json;\nconst repoUrl = `https://git.keruhomelab.com/${p.owner}/${p.repo}`;\nconst prUrl = `${repoUrl}/pulls/${p.prNumber}`;\nlet n;\nif (p.directive === 'MERGE') {\n n = { topic: 'genome-ingest', title: `${p.repo} \\u00b7 PR #${p.prNumber} mergiata`,\n priority: 'default', tags: 'twisted_rightwards_arrows', click: prUrl,\n actions: `view, Vedi la PR, ${prUrl}`,\n body: `PR #${p.prNumber} mergiata su \\`${p.base}\\` da **${p.sender}**.` };\n} else {\n n = { topic: 'genome-ingest', title: `${p.repo} \\u00b7 PR #${p.prNumber} chiusa`,\n priority: 'default', tags: 'wastebasket', click: repoUrl, actions: '',\n body: `**REJECT** di **${p.sender}**: PR #${p.prNumber} chiusa e branch \\`${p.branch}\\` rimosso. Nessun nuovo tentativo.\\n> ${p.feedback}` };\n}\nreturn n;"
},
"id": "f3c339cb-91e3-4436-b56a-b97c81d4a58f",
"name": "Build ntfy action",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1440,
304
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// Security / near-miss: unauthorized sender, invalid directive, or the feat/ guard. On all three\n// paths Switch/Guardia pass the parser output through, so $json carries the directive + context.\nconst d = $json;\nconst repoUrl = (d.owner && d.repo) ? `https://git.keruhomelab.com/${d.owner}/${d.repo}` : '';\nlet n;\nif (d.directive === 'UNAUTHORIZED') {\n n = { topic: 'genome-ingest', title: `Sicurezza \\u00b7 direttiva non autorizzata`,\n priority: 'high', tags: 'no_entry', click: repoUrl, actions: '',\n body: `**${d.sender}** ha tentato \\`${d.attempted}\\` su PR #${d.prNumber}, ma non \\u00e8 tra i maintainer autorizzati. **Nessuna azione** eseguita.` };\n} else if (d.directive === 'INVALID') {\n n = { topic: 'genome-ingest', title: `Direttiva non applicata`,\n priority: 'low', tags: 'information_source', click: repoUrl, actions: '',\n body: `\\`${d.attempted}\\` su PR #${d.prNumber} ignorata: precondizioni non soddisfatte (branch / base / marker raw).` };\n} else {\n n = { topic: 'genome-ingest', title: `Sicurezza \\u00b7 branch protetto`,\n priority: 'high', tags: 'no_entry', click: repoUrl, actions: '',\n body: `Rifiutata azione distruttiva (\\`${d.directive}\\`) sul branch \\`${d.branch}\\`: non \\u00e8 un \\`feat/ai-ingest-*\\`. **Nessuna modifica.**` };\n}\nreturn n;"
},
"id": "ba552761-b8cb-43c8-a6b1-ac93ca2b17b1",
"name": "Build ntfy sicurezza",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
784,
496
]
},
{
"parameters": {
"method": "POST",
"url": "=http://ntfy/{{ $json.topic }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "Title",
"value": "={{ $json.title }}"
},
{
"name": "Priority",
"value": "={{ $json.priority }}"
},
{
"name": "Tags",
"value": "={{ $json.tags }}"
},
{
"name": "Click",
"value": "={{ $json.click }}"
},
{
"name": "Actions",
"value": "={{ $json.actions }}"
},
{
"name": "Markdown",
"value": "yes"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {
"timeout": 15000
}
},
"id": "156f41ad-e1e1-4a7b-b91c-ceb2043ab147",
"name": "ntfy: send",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
1664,
384
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
},
"httpBearerAuth": {
"id": "nCv4CUN7Ef086Ewj",
"name": "Bearer Auth account"
}
}
}
],
"pinData": {},
"connections": {
"Webhook PR Review": {
"main": [
[
{
"node": "Parse & validate",
"type": "main",
"index": 0
}
]
]
},
"Parse & validate": {
"main": [
[
{
"node": "Switch",
"type": "main",
"index": 0
}
]
]
},
"Switch": {
"main": [
[
{
"node": "Forgejo Merge PR",
"type": "main",
"index": 0
}
],
[
{
"node": "Power Manager - ensure-on",
"type": "main",
"index": 0
}
],
[
{
"node": "Guardia feat/",
"type": "main",
"index": 0
}
],
[
{
"node": "Guardia feat/",
"type": "main",
"index": 0
}
],
[
{
"node": "Guardia feat/",
"type": "main",
"index": 0
}
],
[
{
"node": "Build ntfy sicurezza",
"type": "main",
"index": 0
}
],
[
{
"node": "Build ntfy sicurezza",
"type": "main",
"index": 0
}
]
]
},
"Forgejo Merge PR": {
"main": [
[
{
"node": "Build ntfy action",
"type": "main",
"index": 0
}
]
]
},
"Guardia feat/": {
"main": [
[
{
"node": "Forgejo Close PR",
"type": "main",
"index": 0
}
],
[
{
"node": "Build ntfy sicurezza",
"type": "main",
"index": 0
}
]
]
},
"Forgejo Close PR": {
"main": [
[
{
"node": "Forgejo Delete Branch",
"type": "main",
"index": 0
}
]
]
},
"Forgejo Delete Branch": {
"main": [
[
{
"node": "E' REJECT?",
"type": "main",
"index": 0
}
]
]
},
"E' REJECT?": {
"main": [
[
{
"node": "Build ntfy action",
"type": "main",
"index": 0
}
],
[
{
"node": "Power Manager - ensure-on",
"type": "main",
"index": 0
}
]
]
},
"Power Manager - ensure-on": {
"main": [
[
{
"node": "Run one ingest (rework)",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy action": {
"main": [
[
{
"node": "ntfy: send",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy sicurezza": {
"main": [
[
{
"node": "ntfy: send",
"type": "main",
"index": 0
}
]
]
}
},
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate",
"timeSavedMode": "fixed",
"errorWorkflow": "7Vws3gCX3QnjM3oD",
"callerPolicy": "workflowsFromSameOwner",
"availableInMCP": false
},
"versionId": "8c92ff1a-672a-4d15-9aa0-10d5fe11e472",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},
"id": "iho7kFQsXbGIxG7P",
"tags": []
}

View file

@ -7,48 +7,28 @@
"path": "forgejo-push",
"options": {}
},
"id": "eb4abf4a-d26d-4aea-85a2-fc356b81385f",
"name": "Webhook",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2.1,
"position": [
1040,
240
1920,
1728
],
"id": "9cc1b02e-6885-4a19-af34-ed2783ae99bf",
"name": "Webhook",
"webhookId": "bb518834-da85-46bb-bb72-97ba21a78faa"
"webhookId": "cf215f5d31e04dd2"
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose",
"version": 2
"jsCode": "// Bell filter: proceed ONLY on develop pushes that actually touch raw/. Returning [] stops the\n// flow (no node needed). Performance: never wake vm101 for wiki-only pushes (e.g. an ingest PR\n// merged back to develop). pending-raw remains the source of truth.\nconst b = $json.body || $json;\nconst ref = b.ref || '';\nconst genome = (b.repository && b.repository.name) || '';\nif (ref !== 'refs/heads/develop') return [];\nif (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(genome)) return [];\nconst commits = b.commits || [];\nconst touched = [];\nfor (const c of commits) {\n for (const p of (c.added || [])) touched.push(p);\n for (const p of (c.modified || [])) touched.push(p);\n for (const p of (c.removed || [])) touched.push(p);\n}\nif (!touched.some(p => p.startsWith('raw/'))) return [];\nreturn [{ json: { genome } }];"
},
"conditions": [
{
"id": "cc000000-0000-4000-8000-000000000001",
"leftValue": "={{ $json.body.ref }}",
"rightValue": "refs/heads/develop",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"combinator": "and"
},
"options": {}
},
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"id": "190d44ea-4f6f-4cff-91aa-3e65ef44cb21",
"name": "Gate push",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1264,
240
],
"id": "b2dd46aa-cdc3-4103-ad05-c728d9bd14ee",
"name": "IF: ref == develop"
2144,
1728
]
},
{
"parameters": {
@ -84,29 +64,28 @@
},
"options": {}
},
"id": "40af578c-eac4-47ac-9c30-5596eceaf9df",
"name": "Power Manager - ensure-on",
"type": "n8n-nodes-base.executeWorkflow",
"typeVersion": 1.3,
"position": [
1488,
240
],
"id": "e10f6af4-73ac-4689-b9f4-9c656d7c0cc4",
"name": "Power Manager - ensure-on"
2352,
1728
]
},
{
"parameters": {
"authentication": "privateKey",
"command": "=ssh vm101 'pi changed-raw {{ $('Webhook').item.json.body.repository.name }} {{ $('Webhook').item.json.body.before }} {{ $('Webhook').item.json.body.after }}'"
"command": "=ssh vm101 'pi pending-raw {{ $('Gate push').first().json.genome }}'"
},
"id": "f8861a50-aaf1-46fb-95a9-b9b200d4d6ae",
"name": "SSH: pending-raw",
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
1712,
240
2576,
1728
],
"id": "479d2e9d-0fde-417a-9122-d9780cc5dcba",
"name": "SSH: changed-raw",
"executeOnce": true,
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
@ -116,69 +95,166 @@
},
{
"parameters": {
"jsCode": "// run-once-for-all: parse changed-raw (JSON intero) -> 1 item per raw.\n// I nomi raw con spazi o caratteri non sicuri romperebbero il trasporto SSH\n// (lo spazio e' separatore di token nel comando 'pi ingest g raw'). Quindi qui\n// valido i nomi: quelli problematici NON vengono ingeriti, ma emettono un item\n// di tipo 'badname' che a valle diventa un ntfy 'rinomina il file'.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d;\ntry { d = JSON.parse(out); }\ncatch (e) { return [{ json: { _kind: 'error', reason: 'changed-raw non parsabile', raw: out } }]; }\nif (!d.files || d.files.length === 0) return []; // niente raw -> stop silenzioso\n\n// regola 'non rompicoglioni': consentiti lettere, numeri, punto, slash, trattino, underscore.\n// VIETATI: spazi e tutto il resto (che spezzano SSH o gli slug downstream).\nconst SAFE = /^[A-Za-z0-9._\\/-]+$/;\nconst out_items = [];\nfor (const raw of d.files) {\n if (SAFE.test(raw)) {\n out_items.push({ json: { _kind: 'ingest', genome: d.genome, raw } });\n } else {\n out_items.push({ json: { _kind: 'badname', genome: d.genome, raw,\n hint: raw.replace(/[^A-Za-z0-9._\\/-]+/g, '-').toLowerCase() } });\n }\n}\nreturn out_items;"
"jsCode": "// Parse pending-raw -> one item per raw, carrying everything run-one-ingest needs. Unsafe\n// filenames (spaces / odd chars) are NOT ingested -> a 'badname' item -> ntfy.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d;\ntry { d = JSON.parse(out); }\ncatch (e) { return [{ json: { _kind: 'error', reason: 'pending-raw non parsabile', raw: out } }]; }\nif (!d.files || d.files.length === 0) return [];\nconst why = {};\nfor (const it of (d.detail || [])) why[it.path] = it.reason;\nconst SAFE = /^[A-Za-z0-9._\\\\/-]+$/;\nconst items = [];\nfor (const raw of d.files) {\n if (SAFE.test(raw)) items.push({ json: { _kind: 'ingest', genome: d.genome, raw,\n mode: 'ingest', feedback_b64: '', reason: why[raw] || 'new', prevPr: '' } });\n else items.push({ json: { _kind: 'badname', genome: d.genome, raw,\n hint: raw.replace(/[^A-Za-z0-9._\\\\/-]+/g, '-').toLowerCase() } });\n}\nreturn items;"
},
"id": "e1f1e251-1565-4092-b8e7-b97c9c0bb18d",
"name": "Split raw files",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1920,
240
],
"id": "d540e454-4648-475c-8dce-5111ef876f75",
"name": "Split raw files"
2800,
1728
]
},
{
"parameters": {
"authentication": "privateKey",
"command": "=ssh vm101 'pi ingest {{ $json.genome }} {{ $json.raw }}'"
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
2144,
240
"conditions": [
{
"id": "cbacf5d98d594ba5",
"leftValue": "={{ $json._kind }}",
"rightValue": "ingest",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"id": "7e30e055-7bc5-484a-a405-d29ea06ff175",
"name": "SSH: pi ingest",
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
"name": "n8n container -> n8n-runner@nexus"
"combinator": "and"
},
"options": {}
},
"id": "3339ce75-3ec9-4ed0-8faa-2433e9616c43",
"name": "Nome valido?",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
3024,
1728
]
},
{
"parameters": {
"workflowId": {
"__rl": true,
"value": "VIi2ovb5gJxNJLbg",
"mode": "list",
"cachedResultUrl": "/workflow/VIi2ovb5gJxNJLbg",
"cachedResultName": "Genome: run-one-ingest"
},
"workflowInputs": {
"mappingMode": "defineBelow",
"value": {
"genome": "={{ $json.genome }}",
"raw": "={{ $json.raw }}",
"mode": "ingest",
"feedback_b64": "",
"reason": "={{ $json.reason }}",
"prevPr": ""
},
"matchingColumns": [],
"schema": [
{
"id": "genome",
"displayName": "genome",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "raw",
"displayName": "raw",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "mode",
"displayName": "mode",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "feedback_b64",
"displayName": "feedback_b64",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "reason",
"displayName": "reason",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
},
{
"id": "prevPr",
"displayName": "prevPr",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
}
],
"attemptToConvertTypes": false,
"convertFieldsToString": true
},
"options": {
"waitForSubWorkflow": false
}
},
"id": "fda796f5-588b-4502-a653-5d27c3f72ac6",
"name": "Run one ingest",
"type": "n8n-nodes-base.executeWorkflow",
"typeVersion": 1.3,
"position": [
3232,
1616
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// per-item: ultima riga JSON di run-ingest.sh\nconst out = ($json.stdout || '').trim();\nconst line = out.split('\\n').filter(l => l.trim().startsWith('{')).pop();\nif (!line) return { status: 'error', reason: 'nessuna riga JSON run-ingest', raw: out };\ntry { return JSON.parse(line); } catch (e) { return { status: 'error', reason: 'JSON non parsabile', raw: line }; }"
"jsCode": "const d = $json;\nreturn { topic: 'genome-ingest', title: `${d.genome} \\u00b7 file da rinominare`,\n priority: 'high', tags: 'warning', click: '', actions: '',\n body: `Il file \\`${d.raw}\\` ha spazi o caratteri non ammessi e **non** \\u00e8 stato ingerito.\\nRinominalo in: \\`${d.hint}\\`` };"
},
"id": "6820f3fb-97bb-45bf-8e7f-00eb68d7f313",
"name": "Build ntfy badname",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
2368,
240
],
"id": "f60878f4-8cca-43d0-b8b3-0aa1a422237b",
"name": "Parse ingest"
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "const d = $json;\nlet n;\nif (d.status === 'ok') {\n n = { title: `Ingest ${d.slug}: PR aperta`, priority: 'default', tags: 'inbox_tray',\n body: `\\u2705 ${d.slug}: PR aperta (lint ${d.lint_clean ? 'clean' : 'KO'}${d.conflict ? ', CONFLITTO' : ''})\\n\\n\\ud83d\\udd17 ${d.pr_url}` };\n} else if (d.status === 'pr_failed') {\n n = { title: `Ingest ${d.slug}: PR FALLITA`, priority: 'high', tags: 'warning',\n body: `\\u26a0\\ufe0f ${d.slug}: semantic/lint ok ma PR non aperta.\\n\\n${(d.detail || '').split('\\n')[0]}` };\n} else {\n n = { title: 'Ingest: ERRORE', priority: 'high', tags: 'rotating_light',\n body: `\\u274c ${d.reason || 'errore'}\\n\\n${(d.raw || '').slice(0,300)}` };\n}\nreturn n;"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
2592,
240
],
"id": "9018dff6-b314-4ca8-b8ff-fd5423818816",
"name": "Build ntfy"
3232,
1840
]
},
{
"parameters": {
"method": "POST",
"url": "http://ntfy/homelab-genome",
"url": "=http://ntfy/{{ $json.topic }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
@ -195,6 +271,18 @@
{
"name": "Tags",
"value": "={{ $json.tags }}"
},
{
"name": "Click",
"value": "={{ $json.click }}"
},
{
"name": "Actions",
"value": "={{ $json.actions }}"
},
{
"name": "Markdown",
"value": "yes"
}
]
},
@ -202,16 +290,18 @@
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {}
"options": {
"timeout": 15000
}
},
"id": "55cc42b2-3170-4884-bb5d-58e3af97bfea",
"name": "ntfy: send",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
2800,
240
3456,
1840
],
"id": "1f572cb3-741b-46bc-87fa-1e23ade5a9be",
"name": "ntfy: send notification",
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
@ -222,53 +312,6 @@
"name": "Bearer Auth account"
}
}
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "dd000000-0000-4000-8000-000000000001",
"leftValue": "={{ $json._kind }}",
"rightValue": "ingest",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"combinator": "and"
},
"options": {}
},
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
3168,
240
],
"id": "d2d2e2b2-9bd7-446b-b6b8-0a865d49c601",
"name": "IF: nome valido"
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "const d=$json;\nreturn {\n title: 'Ingest: nome file non valido',\n priority: 'high',\n tags: 'warning',\n body: `\\u26a0\\ufe0f \"${d.raw}\" ha spazi o caratteri non ammessi e non e' stato ingerito.\\n\\nRinominalo in: ${d.hint}`\n};"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
3392,
400
],
"id": "3b69aa97-170a-4666-8b0e-4b51b48b2817",
"name": "Build ntfy badname"
}
],
"pinData": {},
@ -277,14 +320,14 @@
"main": [
[
{
"node": "IF: ref == develop",
"node": "Gate push",
"type": "main",
"index": 0
}
]
]
},
"IF: ref == develop": {
"Gate push": {
"main": [
[
{
@ -299,14 +342,14 @@
"main": [
[
{
"node": "SSH: changed-raw",
"node": "SSH: pending-raw",
"type": "main",
"index": 0
}
]
]
},
"SSH: changed-raw": {
"SSH: pending-raw": {
"main": [
[
{
@ -321,51 +364,18 @@
"main": [
[
{
"node": "IF: nome valido",
"node": "Nome valido?",
"type": "main",
"index": 0
}
]
]
},
"SSH: pi ingest": {
"Nome valido?": {
"main": [
[
{
"node": "Parse ingest",
"type": "main",
"index": 0
}
]
]
},
"Parse ingest": {
"main": [
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy": {
"main": [
[
{
"node": "ntfy: send notification",
"type": "main",
"index": 0
}
]
]
},
"IF: nome valido": {
"main": [
[
{
"node": "SSH: pi ingest",
"node": "Run one ingest",
"type": "main",
"index": 0
}
@ -383,7 +393,7 @@
"main": [
[
{
"node": "ntfy: send notification",
"node": "ntfy: send",
"type": "main",
"index": 0
}
@ -394,9 +404,13 @@
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate"
"binaryMode": "separate",
"timeSavedMode": "fixed",
"errorWorkflow": "7Vws3gCX3QnjM3oD",
"callerPolicy": "workflowsFromSameOwner",
"availableInMCP": false
},
"versionId": "2115dd9f-e2b6-4acb-8de0-4a166eb9729a",
"versionId": "d58601e7-b752-4c9f-9438-d91be663c82e",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},

View file

@ -0,0 +1,128 @@
{
"name": "Genome: on-error",
"nodes": [
{
"parameters": {},
"id": "eee467d7-5f8b-4abf-8923-1c70a29dafb2",
"name": "Error Trigger",
"type": "n8n-nodes-base.errorTrigger",
"typeVersion": 1,
"position": [
0,
0
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// Global error handler: set this workflow as the \"Error Workflow\" in each genome workflow's\n// Settings. Catches ANY node failure (SSH down, Forgejo 4xx/5xx, etc.) and notifies once.\nconst e = $json.execution || {};\nconst w = $json.workflow || {};\nconst msg = (e.error && (e.error.message || e.error.description)) || 'errore sconosciuto';\nconst lastNode = (e.lastNodeExecuted) ? ` (nodo: ${e.lastNodeExecuted})` : '';\nreturn { topic: 'genome-ingest', title: `Workflow KO \\u00b7 ${w.name || 'n8n'}`,\n priority: 'high', tags: 'rotating_light',\n click: e.url || '', actions: e.url ? `view, Apri l'esecuzione, ${e.url}` : '',\n body: `**${w.name || 'workflow'}** \\u00e8 fallito${lastNode}.\\n${msg}` };"
},
"id": "bdbf5186-143d-4482-b873-5760fbdabab0",
"name": "Build ntfy",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
240,
0
]
},
{
"parameters": {
"method": "POST",
"url": "=http://ntfy/{{ $json.topic }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "Title",
"value": "={{ $json.title }}"
},
{
"name": "Priority",
"value": "={{ $json.priority }}"
},
{
"name": "Tags",
"value": "={{ $json.tags }}"
},
{
"name": "Click",
"value": "={{ $json.click }}"
},
{
"name": "Actions",
"value": "={{ $json.actions }}"
},
{
"name": "Markdown",
"value": "yes"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {
"timeout": 15000
}
},
"id": "16e9a3af-6acc-46f2-bc56-79e185fddf53",
"name": "ntfy: send",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
464,
0
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
},
"httpBearerAuth": {
"id": "nCv4CUN7Ef086Ewj",
"name": "Bearer Auth account"
}
}
}
],
"pinData": {},
"connections": {
"Error Trigger": {
"main": [
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy": {
"main": [
[
{
"node": "ntfy: send",
"type": "main",
"index": 0
}
]
]
}
},
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate"
},
"versionId": "95bfb02a-7122-43d7-bec6-3a2e5b77a469",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},
"id": "7Vws3gCX3QnjM3oD",
"tags": []
}

View file

@ -0,0 +1,322 @@
{
"name": "Genome: prune",
"nodes": [
{
"parameters": {
"httpMethod": "POST",
"path": "forgejo-push-prune",
"options": {}
},
"id": "050b318b-d3bb-4c1a-baa4-a1e2bd1babd8",
"name": "Webhook prune",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2.1,
"position": [
0,
0
],
"webhookId": "d6ac11900058434e"
},
{
"parameters": {
"jsCode": "// Bell filter for PRUNING: proceed only on develop pushes that REMOVED a raw/ file.\n// Adds/modifications are the ingest flow's job; this flow reacts to deletions only.\nconst b = $json.body || $json;\nconst ref = b.ref || '';\nconst genome = (b.repository && b.repository.name) || '';\nif (ref !== 'refs/heads/develop') return [];\nif (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(genome)) return [];\nconst removed = [];\nfor (const c of (b.commits || [])) for (const p of (c.removed || [])) removed.push(p);\nif (!removed.some(p => p.startsWith('raw/'))) return []; // nothing under raw/ removed -> ignore\nreturn [{ json: { genome } }];"
},
"id": "deac740a-2046-4de8-91eb-812114edeb7b",
"name": "Gate prune",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
224,
0
]
},
{
"parameters": {
"workflowId": {
"__rl": true,
"value": "zbtRXWsLt56nEIfz",
"mode": "list",
"cachedResultUrl": "/workflow/zbtRXWsLt56nEIfz",
"cachedResultName": "Power Manager"
},
"workflowInputs": {
"mappingMode": "defineBelow",
"value": {
"mode": "ensure-on"
},
"matchingColumns": [
"mode"
],
"schema": [
{
"id": "mode",
"displayName": "mode",
"required": false,
"defaultMatch": false,
"display": true,
"canBeUsedToMatch": true,
"type": "string",
"removed": false
}
],
"attemptToConvertTypes": false,
"convertFieldsToString": true
},
"options": {}
},
"id": "554737ea-09f3-4d4b-9a66-c983cc72e655",
"name": "Power Manager - ensure-on",
"type": "n8n-nodes-base.executeWorkflow",
"typeVersion": 1.3,
"position": [
448,
0
]
},
{
"parameters": {
"authentication": "privateKey",
"command": "=ssh vm101 'pi orphan-wiki {{ $('Gate prune').first().json.genome }}'"
},
"id": "311005a4-16fd-4752-92d3-e3bbb9cdf19f",
"name": "SSH: orphan-wiki",
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
672,
0
],
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
"name": "n8n container -> n8n-runner@nexus"
}
}
},
{
"parameters": {
"jsCode": "// Gate: prune only if orphan-wiki found orphans. run-prune re-derives independently anyway\n// (no detected-vs-pruned race) — this just avoids taking the lock for nothing.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d; try { d = JSON.parse(out); } catch (e) { return []; }\nif (!d || !d.count) return []; // 0 orphans -> stop silently\nreturn [{ json: { genome: d.genome, count: d.count } }];"
},
"id": "2374b63c-f0db-4ae9-b350-3fec70687384",
"name": "Orfani?",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
880,
0
]
},
{
"parameters": {
"authentication": "privateKey",
"command": "=ssh vm101 'pi prune {{ $json.genome }}'"
},
"id": "e4130173-ff62-4e11-b3d1-ee7870803663",
"name": "SSH: prune",
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
1104,
0
],
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
"name": "n8n container -> n8n-runner@nexus"
}
}
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "const out = ($json.stdout || '').toString().trim();\nconst line = out.split('\\n').filter(l => l.trim().startsWith('{')).pop();\nlet r; try { r = line ? JSON.parse(line) : { status:'error', reason:'nessuna riga JSON' }; }\ncatch (e) { r = { status:'error', reason:'JSON non parsabile' }; }\nreturn r;"
},
"id": "b649df9a-1e64-49cc-9e7f-6f78a1190382",
"name": "Parse prune",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1328,
0
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// Pruning notification on genome-ingest (the \"system produced a PR to judge\" topic), broom icon.\nconst d = $json;\nconst g = $('Orfani?').first().json;\nlet n;\nif (d.status === 'ok') {\n const pm = (d.pr_url || '').match(/\\/pulls\\/(\\d+)/); const num = pm ? `#${pm[1]}` : '';\n n = { topic:'genome-ingest', title:`${g.genome} \\u00b7 potatura ${num}`.replace(/\\s+/g,' ').trim(),\n priority:'default', tags:'broom', click:d.pr_url || '', actions:d.pr_url ? `view, Apri la PR, ${d.pr_url}` : '',\n body:`${d.count} sorgente/i orfane proposte per la rimozione. **Approva la PR** per potare, oppure chiudila da Forgejo per annullare.` };\n} else {\n n = { topic:'genome-ingest', title:`${g ? g.genome : ''} \\u00b7 errore potatura`.trim(), priority:'high',\n tags:'rotating_light', click:'', actions:'', body:`${d.reason || 'errore'}.` };\n}\nreturn n;"
},
"id": "00be2a1f-6097-40e2-81f5-9dcba40b66ae",
"name": "Build ntfy",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1552,
0
]
},
{
"parameters": {
"method": "POST",
"url": "=http://ntfy/{{ $json.topic }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "Title",
"value": "={{ $json.title }}"
},
{
"name": "Priority",
"value": "={{ $json.priority }}"
},
{
"name": "Tags",
"value": "={{ $json.tags }}"
},
{
"name": "Click",
"value": "={{ $json.click }}"
},
{
"name": "Actions",
"value": "={{ $json.actions }}"
},
{
"name": "Markdown",
"value": "yes"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {
"timeout": 15000
}
},
"id": "50a36840-be50-43ec-bea9-44819a88a923",
"name": "ntfy: send",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
1760,
0
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
},
"httpBearerAuth": {
"id": "nCv4CUN7Ef086Ewj",
"name": "Bearer Auth account"
}
}
}
],
"pinData": {},
"connections": {
"Webhook prune": {
"main": [
[
{
"node": "Gate prune",
"type": "main",
"index": 0
}
]
]
},
"Gate prune": {
"main": [
[
{
"node": "Power Manager - ensure-on",
"type": "main",
"index": 0
}
]
]
},
"Power Manager - ensure-on": {
"main": [
[
{
"node": "SSH: orphan-wiki",
"type": "main",
"index": 0
}
]
]
},
"SSH: orphan-wiki": {
"main": [
[
{
"node": "Orfani?",
"type": "main",
"index": 0
}
]
]
},
"Orfani?": {
"main": [
[
{
"node": "SSH: prune",
"type": "main",
"index": 0
}
]
]
},
"SSH: prune": {
"main": [
[
{
"node": "Parse prune",
"type": "main",
"index": 0
}
]
]
},
"Parse prune": {
"main": [
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy": {
"main": [
[
{
"node": "ntfy: send",
"type": "main",
"index": 0
}
]
]
}
},
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate"
},
"versionId": "ff0be89b-7930-4171-a547-5dc7bffc9472",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},
"id": "smH5Qrv7CQnTtdAF",
"tags": []
}

View file

@ -1,222 +0,0 @@
{
"name": "Genome: raw → commit",
"nodes": [
{
"parameters": {
"rule": {
"interval": [
{
"field": "cronExpression",
"expression": "*/2 * * * *"
}
]
}
},
"type": "n8n-nodes-base.scheduleTrigger",
"typeVersion": 1.3,
"position": [
384,
1056
],
"id": "520c79c8-76e6-41c0-8836-4d8d8f4ed236",
"name": "Schedule: ogni 2 min"
},
{
"parameters": {
"authentication": "privateKey",
"command": "sudo -u homelab -H /usr/local/bin/genome-raw-commit genome-test"
},
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
608,
1056
],
"id": "fe89a85f-d63e-47d9-a7b4-08222f2635d0",
"name": "SSH: genome-raw-commit",
"executeOnce": true,
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
"name": "n8n container -> n8n-runner@nexus"
}
}
},
{
"parameters": {
"jsCode": "// Lo script ora stampa JSON multilinea (jq -n). git manda i progressi su stderr,\n// quindi stdout e' SOLO il JSON: si parsa per intero.\nconst out = ($input.first().json.stdout || '').trim();\nlet data;\ntry {\n data = JSON.parse(out);\n} catch (e) {\n data = { status: 'error', reason: 'output non parsabile', genome: 'genome-test', raw: out };\n}\nreturn [{ json: data }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
832,
1056
],
"id": "74051cc5-5760-453d-80e4-0696d31bfc15",
"name": "Parse result"
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "c0000000-0000-4000-8000-000000000001",
"leftValue": "={{ $json.status }}",
"rightValue": "noop",
"operator": {
"type": "string",
"operation": "notEquals"
}
}
],
"combinator": "and"
},
"options": {}
},
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
1056,
1056
],
"id": "5813753d-f015-4a4e-b386-9d60659077c3",
"name": "IF: non noop"
},
{
"parameters": {
"jsCode": "const d = $input.first().json;\nlet n;\nif (d.status === 'ok') {\n const f = d.files && d.files[0];\n n = {\n title: `Genome: ${d.commits} raw -> ${d.base}`,\n priority: 'default',\n tags: 'inbox_tray',\n body: `✅ ${d.genome}: ${d.commits} commit su ${d.base} (HEAD ${d.head})\\n\\n${d.summary || ''}`\n + (f ? `\\n\\n🔗 Forgejo: ${f.remote_url}\\n📂 Locale: ${f.local_url}` : '')\n };\n} else {\n n = {\n title: 'Genome raw commit: ERRORE',\n priority: 'high',\n tags: 'warning',\n body: `\\u274C ${d.genome || 'genome-test'}: ${d.reason || 'errore sconosciuto'}`\n };\n}\nreturn [{ json: n }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1264,
976
],
"id": "29eee748-4c2d-4e1e-8013-a64bc9cbf816",
"name": "Build ntfy"
},
{
"parameters": {
"method": "POST",
"url": "http://ntfy/homelab-genome",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "Title",
"value": "={{ $json.title }}"
},
{
"name": "Priority",
"value": "={{ $json.priority }}"
},
{
"name": "Tags",
"value": "={{ $json.tags }}"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
1488,
976
],
"id": "d9b6ca21-59ef-44cf-a4f7-a75dcc7eeab4",
"name": "ntfy: send notification",
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
},
"httpBearerAuth": {
"id": "nCv4CUN7Ef086Ewj",
"name": "Bearer Auth account"
}
}
}
],
"pinData": {},
"connections": {
"Schedule: ogni 2 min": {
"main": [
[
{
"node": "SSH: genome-raw-commit",
"type": "main",
"index": 0
}
]
]
},
"SSH: genome-raw-commit": {
"main": [
[
{
"node": "Parse result",
"type": "main",
"index": 0
}
]
]
},
"Parse result": {
"main": [
[
{
"node": "IF: non noop",
"type": "main",
"index": 0
}
]
]
},
"IF: non noop": {
"main": [
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy": {
"main": [
[
{
"node": "ntfy: send notification",
"type": "main",
"index": 0
}
]
]
}
},
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate"
},
"versionId": "9607be0b-cd8c-4e7a-9ddb-63b6ec22b65d",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},
"id": "whyxMpvJMYQ55J1M",
"tags": []
}

View file

@ -0,0 +1,266 @@
{
"name": "Genome: run-one-ingest",
"nodes": [
{
"parameters": {
"inputSource": "passthrough"
},
"id": "70da9144-1147-4cb5-9868-1f5ee2425d4c",
"name": "On ingest request",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"typeVersion": 1.1,
"position": [
-32,
416
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// SECURITY chokepoint: every ingest to vm101 passes here. Re-validate inputs (defense in depth:\n// callers + the SSH wrapper also validate) and assemble the exact command. Charset-validated\n// fields are safe inside the single-quoted remote command -> no shell injection.\nconst d = $json;\nconst genome = (d.genome || '').toString();\nconst raw = (d.raw || '').toString();\nconst mode = (d.mode || 'ingest').toString();\nconst fb = (d.feedback_b64 || '').toString();\n\nconst okGenome = /^[a-z0-9][a-z0-9-]{0,63}$/.test(genome);\nconst okMode = (mode === 'ingest' || mode === 'rework');\nconst okRaw = raw.startsWith('raw/') && !raw.includes('..') && /^[A-Za-z0-9._\\/-]+$/.test(raw);\nconst okFb = (mode === 'ingest') || /^[A-Za-z0-9+/=]+$/.test(fb);\n\nif (!okGenome || !okMode || !okRaw || !okFb) {\n return { _ok: false, genome, mode,\n _reason: `bad input (genome:${okGenome} mode:${okMode} raw:${okRaw} fb:${okFb})` };\n}\nconst ssh_cmd = (mode === 'rework')\n ? `ssh vm101 'pi ingest-rework ${genome} ${raw} ${fb}'`\n : `ssh vm101 'pi ingest ${genome} ${raw}'`;\nreturn { _ok: true, ssh_cmd, genome, raw, mode, reason: d.reason || '', prevPr: d.prevPr || '' };"
},
"id": "551ec0f1-450c-41ce-88a1-8690bc2c1c0b",
"name": "Guard & build cmd",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
192,
416
]
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "4507e3a8b9714c7e",
"leftValue": "={{ $json._ok }}",
"rightValue": true,
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
}
}
],
"combinator": "and"
},
"options": {}
},
"id": "ea4c36ed-c452-406b-9c94-c58fdc69ed20",
"name": "Input valido?",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
416,
416
]
},
{
"parameters": {
"authentication": "privateKey",
"command": "={{ $json.ssh_cmd }}"
},
"id": "a5ea3f08-df3b-4433-a04e-b69ce742575f",
"name": "SSH: ingest",
"type": "n8n-nodes-base.ssh",
"typeVersion": 1,
"position": [
624,
336
],
"credentials": {
"sshPrivateKey": {
"id": "GJQjKzte7Hjdfz89",
"name": "n8n container -> n8n-runner@nexus"
}
}
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// run-ingest.sh prints one JSON line; the wrapper may instead print {status:busy|error,...}.\n// Take the last {...} line from stdout.\nconst out = ($json.stdout || '').toString().trim();\nconst line = out.split('\\n').filter(l => l.trim().startsWith('{')).pop();\nlet r;\ntry { r = line ? JSON.parse(line) : { status: 'error', reason: 'nessuna riga JSON', raw: out }; }\ncatch (e) { r = { status: 'error', reason: 'JSON non parsabile', raw: line }; }\nreturn r;"
},
"id": "0ee5e6c2-111a-4458-aab7-20a683f027ee",
"name": "Parse result",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
848,
336
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// One builder for ingest + rework outcomes. Title is plain ASCII; the icon comes from Tags\n// (ntfy shortcodes); navigation is via Click (tap) + Actions (button) so it works on every\n// client. $('Guard...').item is reliable here: no executeWorkflow sits between Guard and here.\nconst g = $('Guard & build cmd').item.json;\nconst verb = (g.mode === 'rework') ? 'rework' : 'ingest';\nconst d = $json;\nlet n;\nif (g._ok === false) {\n n = { title: `Errore ${verb}: input non valido`, priority: 'high', tags: 'rotating_light',\n click: '', actions: '', body: `Richiesta di ${verb} rifiutata.\\n${g._reason}` };\n} else if (d.status === 'ok') {\n const pm = (d.pr_url || '').match(/\\/pulls\\/(\\d+)/);\n const num = pm ? `#${pm[1]}` : '';\n const lint = d.lint_clean ? 'lint pulito' : 'lint con avvisi';\n const conflict = d.conflict ? ' \\u00b7 \\u26a0\\ufe0f conflitto da risolvere' : '';\n n = { title: `${g.genome} \\u00b7 ${verb} ${d.slug} ${num}`.replace(/\\s+/g,' ').trim(),\n priority: d.conflict ? 'high' : 'default',\n tags: d.conflict ? 'warning' : 'white_check_mark',\n click: d.pr_url || '', actions: d.pr_url ? `view, Apri la PR, ${d.pr_url}` : '',\n body: `**${d.slug}** ${verb === 'rework' ? 'rilavorata' : 'ingerita'}`\n + (g.reason && verb === 'ingest' ? ` (${g.reason})` : '')\n + (g.prevPr ? ` \\u00b7 sostituisce #${g.prevPr}` : '')\n + `.\\n${lint}${conflict}.` };\n} else if (d.status === 'busy') {\n n = { title: `${g.genome} \\u00b7 ${verb} in coda`, priority: 'min', tags: 'hourglass_flowing_sand',\n click: '', actions: '',\n body: `Un altro ingest era in corso su questo genoma. La fonte resta pendente e verr\\u00e0 ripresa al prossimo campanello.` };\n} else if (d.status === 'pr_failed') {\n n = { title: `${g.genome} \\u00b7 ${d.slug}: PR non aperta`, priority: 'high', tags: 'warning',\n click: '', actions: '',\n body: `Semantic e lint ok, ma la PR non si \\u00e8 aperta.\\n${(d.detail || '').split('\\n')[0]}` };\n} else {\n const stage = d.stage ? ` (stage: ${d.stage})` : '';\n n = { title: `${g.genome} \\u00b7 errore ${verb}`, priority: 'high', tags: 'rotating_light',\n click: '', actions: '',\n body: `${(d.reason || 'errore')}${stage}.` + (d.log ? `\\nLog: ${d.log}` : '') };\n}\nn.topic = 'genome-ingest';\nreturn n;"
},
"id": "458318e5-7b26-4695-b564-f58c357d37d0",
"name": "Build ntfy",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1072,
416
]
},
{
"parameters": {
"method": "POST",
"url": "=http://ntfy/{{ $json.topic }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpBearerAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "Title",
"value": "={{ $json.title }}"
},
{
"name": "Priority",
"value": "={{ $json.priority }}"
},
{
"name": "Tags",
"value": "={{ $json.tags }}"
},
{
"name": "Click",
"value": "={{ $json.click }}"
},
{
"name": "Actions",
"value": "={{ $json.actions }}"
},
{
"name": "Markdown",
"value": "yes"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "Raw / Text",
"body": "={{ $json.body }}",
"options": {
"timeout": 15000
}
},
"id": "4a1f0a89-1a56-4e1c-8fbc-173cba4ce97b",
"name": "ntfy: send",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.4,
"position": [
1296,
416
],
"credentials": {
"httpHeaderAuth": {
"id": "TBPXSWOF63k9mvm8",
"name": "ntfy-token"
},
"httpBearerAuth": {
"id": "nCv4CUN7Ef086Ewj",
"name": "Bearer Auth account"
}
}
}
],
"pinData": {},
"connections": {
"On ingest request": {
"main": [
[
{
"node": "Guard & build cmd",
"type": "main",
"index": 0
}
]
]
},
"Guard & build cmd": {
"main": [
[
{
"node": "Input valido?",
"type": "main",
"index": 0
}
]
]
},
"Input valido?": {
"main": [
[
{
"node": "SSH: ingest",
"type": "main",
"index": 0
}
],
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"SSH: ingest": {
"main": [
[
{
"node": "Parse result",
"type": "main",
"index": 0
}
]
]
},
"Parse result": {
"main": [
[
{
"node": "Build ntfy",
"type": "main",
"index": 0
}
]
]
},
"Build ntfy": {
"main": [
[
{
"node": "ntfy: send",
"type": "main",
"index": 0
}
]
]
}
},
"active": true,
"settings": {
"executionOrder": "v1",
"binaryMode": "separate",
"timeSavedMode": "fixed",
"errorWorkflow": "7Vws3gCX3QnjM3oD",
"callerPolicy": "workflowsFromSameOwner",
"availableInMCP": false
},
"versionId": "5d2cf4bd-f2c6-41fc-98a5-eaa797e31417",
"meta": {
"instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417"
},
"id": "VIi2ovb5gJxNJLbg",
"tags": []
}

View file

@ -79,6 +79,29 @@ case "$cmd" in
# MECHANICAL step: validate manifest -> index/log/scoped-lint/commit/PR -> 1 JSON line
exec "${HOME}/.pi/agent/skills/ingest/scripts/run-ingest.sh" "${genome}"
;;
"pi prune "*)
# Pota le source orfane. Stesso lock dell'ingest (serializza le scritture per genoma),
# clean_start, poi run-prune.sh (che ri-deriva gli orfani e apre una PR gated).
genome="${cmd#pi prune }"
case "$genome" in ""|*[!a-z0-9-]*) echo '{"status":"error","reason":"invalid genome name"}'; exit 1;; esac
logger -t n8n-pi-wrap "ok: pi prune ${genome}"
exec 9>"/run/lock/kg-ingest-${genome}.lock" 2>/dev/null || exec 9>"/tmp/kg-ingest-${genome}.lock"
if ! flock -n 9; then
echo '{"status":"busy","reason":"another ingest/prune is running for this genome","genome":"'"$genome"'"}'
exit 0
fi
set -a; . "${HOME}/.config/knowledge-genome.env"; set +a
cd "${GENOMES_ROOT}/${genome}" || { echo '{"status":"error","reason":"unknown genome"}'; exit 1; }
: "${KG_LIB_DIR:=${HOME}/knowledge-genome-orchestrator/lib}"
source "${KG_LIB_DIR}/clean-start.sh" 2>/dev/null \
|| { echo '{"status":"error","reason":"clean-start.sh not found"}'; exit 1; }
clean_start || { echo '{"status":"error","reason":"clean-start failed"}'; exit 1; }
exec "${HOME}/.pi/agent/skills/ingest/scripts/run-prune.sh" "${genome}"
;;
"pi ingest-rework "*)
# args: <genome> <raw_path> <feedback_base64> (3 token).
# Feedback in base64 nell'argv: il nodo SSH di n8n non passa stdin, e cosi' i metacaratteri

View file

@ -1,11 +1,12 @@
#!/usr/bin/env python3
# =============================================================================
# skills/ingest/scripts/index-append.py
# Insert an entry line into the correct section of wiki/index.md and keep that
# section's entries alphabetically ordered. Bumps frontmatter last_updated.
# Insert OR remove an entry line in wiki/index.md, keeping the target section
# alphabetically ordered. Bumps frontmatter last_updated.
#
# index-append.py --section Sources \
# --entry '- [[sources/foo]] — One-line summary. `maturity: draft`'
# index-append.py --remove 'sources/foo' # delete the entry by wikilink
# =============================================================================
import argparse
import datetime
@ -17,14 +18,116 @@ LINK_RE = re.compile(r"^- \[\[([^\]]+)\]\]")
HEADER_RE = re.compile(r"^## ")
def bump_last_updated(lines, today):
"""Bump (or self-heal) last_updated inside the first frontmatter block."""
fm_open = False
fm_close_idx = None
bumped = False
for i, ln in enumerate(lines):
if ln.strip() == "---":
if not fm_open:
fm_open = True
continue
fm_close_idx = i
break
if fm_open and ln.startswith("last_updated:"):
lines[i] = f"last_updated: {today}"
bumped = True
if not fm_open:
print("index-append: warning: no frontmatter found, last_updated not bumped",
file=sys.stderr)
elif not bumped and fm_close_idx is not None:
lines.insert(fm_close_idx, f"last_updated: {today}")
print("index-append: last_updated key was missing — inserted", file=sys.stderr)
def do_remove(lines, link, today):
"""Remove every entry line whose wikilink == link. Idempotent."""
bump_last_updated(lines, today)
kept = []
removed = 0
for ln in lines:
m = LINK_RE.match(ln)
if m and m.group(1) == link:
removed += 1
continue
kept.append(ln)
if removed:
print(f"index-append: removed [[{link}]] ({removed} line(s))")
else:
# Idempotent: the goal state (entry absent) already holds.
print(f"index-append: [[{link}]] not present, nothing to remove")
return kept
def do_append(lines, section, entry, today):
bump_last_updated(lines, today)
# Locate the target section [start, end)
start = None
for i, ln in enumerate(lines):
if HEADER_RE.match(ln) and ln[3:].startswith(section):
start = i
break
if start is None:
print(f"index-append: section '{section}' not found", file=sys.stderr)
return None
end = len(lines)
for i in range(start + 1, len(lines)):
if HEADER_RE.match(lines[i]):
end = i
break
body = lines[start + 1:end]
intro = [ln for ln in body if not ENTRY_RE.match(ln)]
entries = [ln for ln in body if ENTRY_RE.match(ln)]
new_m = LINK_RE.match(entry)
new_link = new_m.group(1) if new_m else None
if new_link is not None:
replaced = False
for idx, ln in enumerate(entries):
m = LINK_RE.match(ln)
if m and m.group(1) == new_link:
if ln == entry:
print("index-append: entry already present, skipping")
return lines
entries[idx] = entry
replaced = True
break
if not replaced:
entries.append(entry)
else:
if entry in entries:
print("index-append: entry already present, skipping")
return lines
entries.append(entry)
entries.sort(key=str.casefold)
while intro and intro[-1].strip() == "":
intro.pop()
new_section = intro + [""] + entries + [""]
print(f"index-append: added to {section}")
return lines[:start + 1] + new_section + lines[end:]
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--section", required=True,
help="Section name, e.g. Sources / Entities / Concepts / Queries / Conflicts")
ap.add_argument("--entry", required=True, help="Full index line to insert")
ap.add_argument("--section", help="Section name (required with --entry)")
ap.add_argument("--entry", help="Full index line to insert")
ap.add_argument("--remove", metavar="WIKILINK",
help="Remove the entry with this wikilink, e.g. sources/foo")
ap.add_argument("--file", default="wiki/index.md")
args = ap.parse_args()
if bool(args.remove) == bool(args.entry):
print("index-append: provide exactly one of --entry or --remove", file=sys.stderr)
return 2
if args.entry and not args.section:
print("index-append: --entry requires --section", file=sys.stderr)
return 2
try:
with open(args.file, encoding="utf-8") as fh:
lines = fh.read().splitlines()
@ -33,90 +136,15 @@ def main() -> int:
return 1
today = datetime.date.today().isoformat()
# 1. Bump last_updated inside the first frontmatter block
fm_open = False
fm_close_idx = None
bumped = False
for i, ln in enumerate(lines):
if ln.strip() == "---":
if not fm_open:
fm_open = True
continue
fm_close_idx = i # the closing ---
break
if fm_open and ln.startswith("last_updated:"):
lines[i] = f"last_updated: {today}"
bumped = True
if not fm_open:
print("index-append: warning: no frontmatter found, last_updated not bumped",
file=sys.stderr)
elif not bumped and fm_close_idx is not None:
# self-heal: frontmatter present but missing the key — insert it before the close
lines.insert(fm_close_idx, f"last_updated: {today}")
print("index-append: last_updated key was missing — inserted", file=sys.stderr)
# 2. Locate the target section [start, end)
start = None
for i, ln in enumerate(lines):
if HEADER_RE.match(ln) and ln[3:].startswith(args.section):
start = i
break
if start is None:
print(f"index-append: section '{args.section}' not found in {args.file}",
file=sys.stderr)
if args.remove:
out = do_remove(lines, args.remove, today)
else:
out = do_append(lines, args.section, args.entry, today)
if out is None:
return 1
end = len(lines)
for i in range(start + 1, len(lines)):
if HEADER_RE.match(lines[i]):
end = i
break
# 3. Split the section body into intro (non-entry) and entries
body = lines[start + 1:end]
intro = [ln for ln in body if not ENTRY_RE.match(ln)]
entries = [ln for ln in body if ENTRY_RE.match(ln)]
# Deduplicate by wikilink PATH, not by exact line: a re-ingest with a changed
# summary/maturity should UPDATE the existing entry, not add a duplicate line.
new_m = LINK_RE.match(args.entry)
new_link = new_m.group(1) if new_m else None
if new_link is not None:
replaced = False
for idx, ln in enumerate(entries):
m = LINK_RE.match(ln)
if m and m.group(1) == new_link:
if ln == args.entry:
print("index-append: entry already present, skipping")
return 0
entries[idx] = args.entry # same page, refreshed text
replaced = True
break
if not replaced:
entries.append(args.entry)
else:
# No parseable wikilink — fall back to exact-line dedup.
if args.entry in entries:
print("index-append: entry already present, skipping")
return 0
entries.append(args.entry)
entries.sort(key=str.casefold)
# Normalise intro: drop trailing blanks, keep header + comment(s)
while intro and intro[-1].strip() == "":
intro.pop()
new_section = intro + [""] + entries + [""]
lines = lines[:start + 1] + new_section + lines[end:]
with open(args.file, "w", encoding="utf-8") as fh:
fh.write("\n".join(lines) + "\n")
print(f"index-append: added to {args.section}")
fh.write("\n".join(out) + "\n")
return 0

View file

@ -60,6 +60,15 @@ with open(raw_rel, "r", encoding="utf-8") as fh:
if not source_text.strip():
die("preflight", "source is empty: " + raw_rel)
# --- pre-flight check: if the prompt exceeds context window, exit cleanly with stage:input ---
# Conservative estimate: ~4 chars/token for mixed IT/EN text
SAFETY_MARGIN = 4096 # room for system prompt + JSON response
MAX_SOURCE_TOKENS = NUM_CTX - SAFETY_MARGIN
MAX_SOURCE_CHARS = MAX_SOURCE_TOKENS * 4
if len(source_text) > MAX_SOURCE_CHARS:
die("input", f"source too large ({len(source_text)} chars, limit ~{MAX_SOURCE_CHARS}). "
f"Use the SPLIT directive or divide the document.")
# --- read existing index to avoid duplicate slugs ---
existing_entities = set()

View file

@ -16,10 +16,11 @@ set -euo pipefail
: "${FORGEJO_USER:?missing FORGEJO_USER}"
: "${FORGEJO_TOKEN:?missing FORGEJO_TOKEN}"
slug="" title="" body_file="" base="main" label=""
slug="" title="" body_file="" base="main" label="" branch=""
while [[ $# -gt 0 ]]; do
case "$1" in
--slug) slug="$2"; shift 2 ;;
--branch) branch="$2"; shift 2 ;;
--title) title="$2"; shift 2 ;;
--body-file) body_file="$2"; shift 2 ;;
--base) base="$2"; shift 2 ;;
@ -28,16 +29,23 @@ while [[ $# -gt 0 ]]; do
esac
done
: "${slug:?--slug required}"
: "${title:?--title required}"
: "${body_file:?--body-file required}"
[[ -f "$body_file" ]] || { echo "open-pr: body file not found: $body_file" >&2; exit 1; }
# --branch overrides the default; otherwise derive the ingest branch from --slug.
# (run-prune passes its own chore/prune-orphans-* branch; run-ingest passes --slug.)
if [[ -z "$branch" ]]; then
: "${slug:?--slug or --branch required}"
branch="feat/ai-ingest-${slug}"
fi
repo="$(basename -s .git "$(git config --get remote.origin.url)")"
# 1. Branch + commit + push (AGENTS.md rule 5: never commit to main)
git switch -c "$branch" 2>/dev/null || git switch "$branch"
# Rolling PR: -C force-resets the branch label to the current base (we are on it after
# clean_start) and CARRIES the freshly-written wiki/ changes, so a re-ingest of the same
# source rebuilds the branch cleanly instead of hitting a dirty-switch refusal.
git switch -C "$branch"
git add wiki/
# Scope BOTH the emptiness check and the commit to wiki/ — never commit anything that
# happened to be staged outside wiki/ (a stray hook, an aborted prior run, etc.).
@ -46,7 +54,10 @@ if git diff --cached --quiet -- wiki/; then
exit 1
fi
git commit -m "$title" -- wiki/
git push -u origin "$branch"
# Try a normal push (new branch / fast-forward). If the branch was rebuilt from base and
# diverged, force-with-lease updates the open PR in place — the lease refuses to clobber if
# origin moved unexpectedly since our fetch, so concurrent work is never lost.
git push -u origin "$branch" 2>/dev/null || git push -u --force-with-lease origin "$branch"
# DRY_RUN: local git work done; skip the Forgejo API (offline tests).
if [[ -n "${DRY_RUN:-}" ]]; then

View file

@ -0,0 +1,96 @@
#!/usr/bin/env bash
# =============================================================================
# skills/ingest/scripts/run-prune.sh
# Symmetric companion to run-ingest: prune source pages whose raw source no
# longer exists. RE-DERIVES the orphan set itself (mirrors orphan-wiki.sh) — it
# never trusts a list handed in by n8n, so there is no "detected-vs-pruned"
# race. Removes ONLY the pages it derived plus their index entries, commits
# ONLY wiki/ on chore/prune-orphans-<date>, and opens a GATED removal PR (the
# operator approves the deletion; principle 2). Never deletes of its own accord.
#
# Runs OUTSIDE the model, on vm101, cwd = genome checkout. The wrapper (`pi
# prune`) has already taken the per-genome lock and done clean_start, exactly
# like `pi ingest` — so this script does neither.
#
# run-prune.sh <genome>
#
# Emits a single JSON result line on stdout for n8n to parse.
# =============================================================================
set -euo pipefail
genome="${1:?usage: run-prune.sh <genome>}"
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
fail() {
jq -nc --arg stage "$1" --arg reason "$2" '{status:"error", stage:$stage, reason:$reason}'
exit 1
}
command -v jq >/dev/null 2>&1 || { echo '{"status":"error","reason":"jq missing"}'; exit 1; }
command -v python3 >/dev/null 2>&1 || fail "deps" "python3 missing (needed by index-append.py)"
# --- re-derive orphans (same rule as orphan-wiki.sh; computed fresh, here, now) ---
# A wiki/sources/*.md page is orphaned when its frontmatter source_path points at
# a raw file that no longer exists. Legacy pages without source_path are ignored.
declare -a ORPH=()
for page in wiki/sources/*.md; do
[[ -e "$page" ]] || continue
sp="$(sed -n 's/^source_path:[[:space:]]*//p' "$page" | tr -d '\r' | head -n1)"
[[ -n "$sp" ]] || continue
[[ -f "$sp" ]] || ORPH+=("$page")
done
if [[ ${#ORPH[@]} -eq 0 ]]; then
jq -nc '{status:"ok", count:0, pruned:[], detail:"no orphans"}'
exit 0
fi
# --- remove each orphan page + its index entry (anti-traversal, wiki/-only) ---
declare -a PRUNED=()
for page in "${ORPH[@]}"; do
case "$page" in
wiki/*) : ;;
*) fail "prune" "refusing to remove outside wiki/: ${page}" ;;
esac
case "$page" in *..*) fail "prune" "path traversal in page: ${page}" ;; esac
[[ -f "$page" ]] || continue
rm -f "$page"
link="${page#wiki/}"; link="${link%.md}" # e.g. sources/foo
python3 "${SCRIPTS}/index-append.py" --remove "$link" \
|| fail "index" "index-append --remove failed for ${link}"
PRUNED+=("$link")
done
# --- assemble the PR body ---
date_tag="$(date +%F)"
body="$(mktemp)"
trap 'rm -f "$body"' EXIT
{
echo "## Prune orphaned sources"
echo ""
echo "These source pages reference a \`source_path\` whose raw file no longer exists"
echo "in \`raw/\`. Removing them keeps the wiki in sync with git (the source of truth)."
echo ""
echo "| Removed page |"
echo "|--------------|"
for l in "${PRUNED[@]}"; do echo "| \`wiki/${l}.md\` |"; done
} > "$body"
# --- open the GATED removal PR on a chore/ branch (open-pr --branch override) ---
branch="chore/prune-orphans-${date_tag}"
pr_out="$( bash "${SCRIPTS}/open-pr.sh" \
--branch "$branch" \
--title "chore: prune ${#PRUNED[@]} orphaned source(s)" \
--body-file "$body" --base "${INGEST_BASE:-main}" 2>&1 )" && pr_rc=0 || pr_rc=$?
pr_url="$(printf '%s\n' "$pr_out" | sed -n 's/^PR opened: //p' | head -n1)"
# --- result line for n8n ---
jq -nc \
--arg status "$([[ $pr_rc -eq 0 ]] && echo ok || echo pr_failed)" \
--argjson count "${#PRUNED[@]}" \
--arg pr_url "$pr_url" \
--arg detail "$pr_out" \
--argjson pruned "$(printf '%s\n' "${PRUNED[@]}" | jq -R . | jq -s .)" \
'{status:$status, count:$count, pr_url:$pr_url, pruned:$pruned, detail:$detail}'
[[ $pr_rc -eq 0 ]] || exit 1

44
tests/index-remove.bats Normal file
View file

@ -0,0 +1,44 @@
#!/usr/bin/env bats
# tests/index-remove.bats — index-append.py --remove mode.
setup() {
load 'helpers'
export GENOMES_ROOT="${BATS_TEST_TMPDIR}"
g_src="$(make_fixture_genome)"; export g="$g_src"
}
@test "index --remove: deletes the matching entry, keeps the others" {
cd "$g"
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — A. `maturity: draft`'
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/b]] — B. `maturity: draft`'
grep -q 'sources/a' wiki/index.md
grep -q 'sources/b' wiki/index.md
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/a'
[ "$status" -eq 0 ]
! grep -q '\[\[sources/a\]\]' wiki/index.md
grep -q 'sources/b' wiki/index.md
}
@test "index --remove: idempotent when the entry is absent" {
cd "$g"
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/does-not-exist'
[ "$status" -eq 0 ]
[[ "$output" == *'nothing to remove'* ]]
}
@test "index --remove: bumps last_updated" {
cd "$g"
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — A. `maturity: draft`'
# set last_updated to an old date, then remove and check it moved
sed -i 's/^last_updated:.*/last_updated: 2000-01-01/' wiki/index.md
run python3 "$SKILL_SCRIPTS/index-append.py" --remove 'sources/a'
[ "$status" -eq 0 ]
! grep -q '2000-01-01' wiki/index.md
grep -q "last_updated: $(date +%F)" wiki/index.md
}
@test "index --remove: rejects passing both --entry and --remove" {
cd "$g"
run python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/a]] — x' --remove 'sources/a'
[ "$status" -eq 2 ]
}

68
tests/run-prune.bats Normal file
View file

@ -0,0 +1,68 @@
#!/usr/bin/env bats
# tests/run-prune.bats — prune orphaned sources (no LLM, no network; DRY_RUN).
setup() {
load 'helpers'
export PRUNE="${SKILL_SCRIPTS}/run-prune.sh"
export GENOMES_ROOT="${BATS_TEST_TMPDIR}"
export INGEST_BASE="main"
export KG_LIB_DIR="${LIB_DIR}"
export FORGEJO_URL="http://forgejo.local" FORGEJO_USER="u" FORGEJO_TOKEN="t"
export DRY_RUN=1
g_src="$(make_fixture_genome)"; export g_name="fixture-genome"
mv "$g_src" "${GENOMES_ROOT}/${g_name}"; export g="${GENOMES_ROOT}/${g_name}"
( cd "$g" && rm -f raw/articles/test.md && git add -A && git commit -q -m clear && git push -q )
}
@test "run-prune: removes only the orphaned source + its index entry, opens a dry PR" {
command -v jq >/dev/null 2>&1 || skip "jq not installed"
cd "$g"
# kept: raw exists. orphan: raw missing.
echo content > raw/articles/kept.md
h="$(sha256sum raw/articles/kept.md | cut -d' ' -f1)"
printf -- '---\nsource_path: raw/articles/kept.md\nsource_sha256: %s\n---\nbody\n' "$h" > wiki/sources/kept.md
printf -- '---\nsource_path: raw/articles/gone.md\nsource_sha256: abc\n---\nbody\n' > wiki/sources/orphan.md
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/kept]] — kept. `maturity: draft`'
python3 "$SKILL_SCRIPTS/index-append.py" --section Sources --entry '- [[sources/orphan]] — orphan. `maturity: draft`'
git add -A && git commit -q -m setup && git push -q
run bash "$PRUNE" "$g_name"
[ "$status" -eq 0 ]
[[ "$output" == *'"status":"ok"'* ]]
[[ "$output" == *'"count":1'* ]]
# only the orphan page is gone
[ ! -f wiki/sources/orphan.md ]
[ -f wiki/sources/kept.md ]
# index reflects the removal
! grep -q 'sources/orphan' wiki/index.md
grep -q 'sources/kept' wiki/index.md
# committed on a chore/ branch (NOT feat/ai-ingest-*)
git rev-parse --verify "chore/prune-orphans-$(date +%F)"
}
@test "run-prune: no orphans -> count 0 and no PR/branch" {
command -v jq >/dev/null 2>&1 || skip "jq not installed"
cd "$g"
echo content > raw/articles/kept.md
h="$(sha256sum raw/articles/kept.md | cut -d' ' -f1)"
printf -- '---\nsource_path: raw/articles/kept.md\nsource_sha256: %s\n---\nbody\n' "$h" > wiki/sources/kept.md
git add -A && git commit -q -m setup && git push -q
run bash "$PRUNE" "$g_name"
[ "$status" -eq 0 ]
[[ "$output" == *'"count":0'* ]]
run git rev-parse --verify "chore/prune-orphans-$(date +%F)"
[ "$status" -ne 0 ]
}
@test "run-prune: refuses when an orphan path would escape wiki/ (defense in depth)" {
command -v jq >/dev/null 2>&1 || skip "jq not installed"
cd "$g"
# legacy page without source_path is ignored; a page with a missing raw is the orphan.
printf -- '---\nsource_path: raw/articles/gone.md\nsource_sha256: abc\n---\nbody\n' > wiki/sources/orphan.md
git add -A && git commit -q -m setup && git push -q
run bash "$PRUNE" "$g_name"
[ "$status" -eq 0 ]
[[ "$output" == *'"count":1'* ]]
[ ! -f wiki/sources/orphan.md ]
}