From 111ffd266ad7818c8c980b4f904eb842da045ed0 Mon Sep 17 00:00:00 2001 From: Matteo Cherubini Date: Wed, 1 Jul 2026 18:24:19 +0200 Subject: [PATCH] refactor(genome-ingest): Use `run-one-ingest` and improve filtering --- deploy/n8n/genome-ingest.json | 374 ++++++++++++++++++---------------- 1 file changed, 194 insertions(+), 180 deletions(-) diff --git a/deploy/n8n/genome-ingest.json b/deploy/n8n/genome-ingest.json index 142f3b4..e5d445c 100644 --- a/deploy/n8n/genome-ingest.json +++ b/deploy/n8n/genome-ingest.json @@ -7,48 +7,28 @@ "path": "forgejo-push", "options": {} }, + "id": "eb4abf4a-d26d-4aea-85a2-fc356b81385f", + "name": "Webhook", "type": "n8n-nodes-base.webhook", "typeVersion": 2.1, "position": [ - 1040, - 240 + 1920, + 1728 ], - "id": "9cc1b02e-6885-4a19-af34-ed2783ae99bf", - "name": "Webhook", - "webhookId": "bb518834-da85-46bb-bb72-97ba21a78faa" + "webhookId": "cf215f5d31e04dd2" }, { "parameters": { - "conditions": { - "options": { - "caseSensitive": true, - "leftValue": "", - "typeValidation": "loose", - "version": 2 - }, - "conditions": [ - { - "id": "cc000000-0000-4000-8000-000000000001", - "leftValue": "={{ $json.body.ref }}", - "rightValue": "refs/heads/develop", - "operator": { - "type": "string", - "operation": "equals" - } - } - ], - "combinator": "and" - }, - "options": {} + "jsCode": "// Bell filter: proceed ONLY on develop pushes that actually touch raw/. Returning [] stops the\n// flow (no node needed). Performance: never wake vm101 for wiki-only pushes (e.g. an ingest PR\n// merged back to develop). pending-raw remains the source of truth.\nconst b = $json.body || $json;\nconst ref = b.ref || '';\nconst genome = (b.repository && b.repository.name) || '';\nif (ref !== 'refs/heads/develop') return [];\nif (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(genome)) return [];\nconst commits = b.commits || [];\nconst touched = [];\nfor (const c of commits) {\n for (const p of (c.added || [])) touched.push(p);\n for (const p of (c.modified || [])) touched.push(p);\n for (const p of (c.removed || [])) touched.push(p);\n}\nif (!touched.some(p => p.startsWith('raw/'))) return [];\nreturn [{ json: { genome } }];" }, - "type": "n8n-nodes-base.if", - "typeVersion": 2.2, + "id": "190d44ea-4f6f-4cff-91aa-3e65ef44cb21", + "name": "Gate push", + "type": "n8n-nodes-base.code", + "typeVersion": 2, "position": [ - 1264, - 240 - ], - "id": "b2dd46aa-cdc3-4103-ad05-c728d9bd14ee", - "name": "IF: ref == develop" + 2144, + 1728 + ] }, { "parameters": { @@ -84,29 +64,28 @@ }, "options": {} }, + "id": "40af578c-eac4-47ac-9c30-5596eceaf9df", + "name": "Power Manager - ensure-on", "type": "n8n-nodes-base.executeWorkflow", "typeVersion": 1.3, "position": [ - 1488, - 240 - ], - "id": "e10f6af4-73ac-4689-b9f4-9c656d7c0cc4", - "name": "Power Manager - ensure-on" + 2352, + 1728 + ] }, { "parameters": { "authentication": "privateKey", - "command": "=ssh vm101 'pi changed-raw {{ $('Webhook').item.json.body.repository.name }} {{ $('Webhook').item.json.body.before }} {{ $('Webhook').item.json.body.after }}'" + "command": "=ssh vm101 'pi pending-raw {{ $('Gate push').first().json.genome }}'" }, + "id": "f8861a50-aaf1-46fb-95a9-b9b200d4d6ae", + "name": "SSH: pending-raw", "type": "n8n-nodes-base.ssh", "typeVersion": 1, "position": [ - 1712, - 240 + 2576, + 1728 ], - "id": "479d2e9d-0fde-417a-9122-d9780cc5dcba", - "name": "SSH: changed-raw", - "executeOnce": true, "credentials": { "sshPrivateKey": { "id": "GJQjKzte7Hjdfz89", @@ -116,69 +95,166 @@ }, { "parameters": { - "jsCode": "// run-once-for-all: parse changed-raw (JSON intero) -> 1 item per raw.\n// I nomi raw con spazi o caratteri non sicuri romperebbero il trasporto SSH\n// (lo spazio e' separatore di token nel comando 'pi ingest g raw'). Quindi qui\n// valido i nomi: quelli problematici NON vengono ingeriti, ma emettono un item\n// di tipo 'badname' che a valle diventa un ntfy 'rinomina il file'.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d;\ntry { d = JSON.parse(out); }\ncatch (e) { return [{ json: { _kind: 'error', reason: 'changed-raw non parsabile', raw: out } }]; }\nif (!d.files || d.files.length === 0) return []; // niente raw -> stop silenzioso\n\n// regola 'non rompicoglioni': consentiti lettere, numeri, punto, slash, trattino, underscore.\n// VIETATI: spazi e tutto il resto (che spezzano SSH o gli slug downstream).\nconst SAFE = /^[A-Za-z0-9._\\/-]+$/;\nconst out_items = [];\nfor (const raw of d.files) {\n if (SAFE.test(raw)) {\n out_items.push({ json: { _kind: 'ingest', genome: d.genome, raw } });\n } else {\n out_items.push({ json: { _kind: 'badname', genome: d.genome, raw,\n hint: raw.replace(/[^A-Za-z0-9._\\/-]+/g, '-').toLowerCase() } });\n }\n}\nreturn out_items;" + "jsCode": "// Parse pending-raw -> one item per raw, carrying everything run-one-ingest needs. Unsafe\n// filenames (spaces / odd chars) are NOT ingested -> a 'badname' item -> ntfy.\nconst out = ($input.first().json.stdout || '').toString().trim();\nlet d;\ntry { d = JSON.parse(out); }\ncatch (e) { return [{ json: { _kind: 'error', reason: 'pending-raw non parsabile', raw: out } }]; }\nif (!d.files || d.files.length === 0) return [];\nconst why = {};\nfor (const it of (d.detail || [])) why[it.path] = it.reason;\nconst SAFE = /^[A-Za-z0-9._\\\\/-]+$/;\nconst items = [];\nfor (const raw of d.files) {\n if (SAFE.test(raw)) items.push({ json: { _kind: 'ingest', genome: d.genome, raw,\n mode: 'ingest', feedback_b64: '', reason: why[raw] || 'new', prevPr: '' } });\n else items.push({ json: { _kind: 'badname', genome: d.genome, raw,\n hint: raw.replace(/[^A-Za-z0-9._\\\\/-]+/g, '-').toLowerCase() } });\n}\nreturn items;" }, + "id": "e1f1e251-1565-4092-b8e7-b97c9c0bb18d", + "name": "Split raw files", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ - 1920, - 240 - ], - "id": "d540e454-4648-475c-8dce-5111ef876f75", - "name": "Split raw files" + 2800, + 1728 + ] }, { "parameters": { - "authentication": "privateKey", - "command": "=ssh vm101 'pi ingest {{ $json.genome }} {{ $json.raw }}'" + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [ + { + "id": "cbacf5d98d594ba5", + "leftValue": "={{ $json._kind }}", + "rightValue": "ingest", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + }, + "options": {} }, - "type": "n8n-nodes-base.ssh", - "typeVersion": 1, + "id": "3339ce75-3ec9-4ed0-8faa-2433e9616c43", + "name": "Nome valido?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, "position": [ - 2144, - 240 - ], - "id": "7e30e055-7bc5-484a-a405-d29ea06ff175", - "name": "SSH: pi ingest", - "credentials": { - "sshPrivateKey": { - "id": "GJQjKzte7Hjdfz89", - "name": "n8n container -> n8n-runner@nexus" + 3024, + 1728 + ] + }, + { + "parameters": { + "workflowId": { + "__rl": true, + "value": "VIi2ovb5gJxNJLbg", + "mode": "list", + "cachedResultUrl": "/workflow/VIi2ovb5gJxNJLbg", + "cachedResultName": "Genome: run-one-ingest" + }, + "workflowInputs": { + "mappingMode": "defineBelow", + "value": { + "genome": "={{ $json.genome }}", + "raw": "={{ $json.raw }}", + "mode": "ingest", + "feedback_b64": "", + "reason": "={{ $json.reason }}", + "prevPr": "" + }, + "matchingColumns": [], + "schema": [ + { + "id": "genome", + "displayName": "genome", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + }, + { + "id": "raw", + "displayName": "raw", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + }, + { + "id": "mode", + "displayName": "mode", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + }, + { + "id": "feedback_b64", + "displayName": "feedback_b64", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + }, + { + "id": "reason", + "displayName": "reason", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + }, + { + "id": "prevPr", + "displayName": "prevPr", + "required": false, + "defaultMatch": false, + "display": true, + "canBeUsedToMatch": true, + "type": "string", + "removed": false + } + ], + "attemptToConvertTypes": false, + "convertFieldsToString": true + }, + "options": { + "waitForSubWorkflow": false } - } + }, + "id": "fda796f5-588b-4502-a653-5d27c3f72ac6", + "name": "Run one ingest", + "type": "n8n-nodes-base.executeWorkflow", + "typeVersion": 1.3, + "position": [ + 3232, + 1616 + ] }, { "parameters": { "mode": "runOnceForEachItem", - "jsCode": "// per-item: ultima riga JSON di run-ingest.sh\nconst out = ($json.stdout || '').trim();\nconst line = out.split('\\n').filter(l => l.trim().startsWith('{')).pop();\nif (!line) return { status: 'error', reason: 'nessuna riga JSON run-ingest', raw: out };\ntry { return JSON.parse(line); } catch (e) { return { status: 'error', reason: 'JSON non parsabile', raw: line }; }" + "jsCode": "const d = $json;\nreturn { topic: 'genome-ingest', title: `${d.genome} \\u00b7 file da rinominare`,\n priority: 'high', tags: 'warning', click: '', actions: '',\n body: `Il file \\`${d.raw}\\` ha spazi o caratteri non ammessi e **non** \\u00e8 stato ingerito.\\nRinominalo in: \\`${d.hint}\\`` };" }, + "id": "6820f3fb-97bb-45bf-8e7f-00eb68d7f313", + "name": "Build ntfy badname", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ - 2368, - 240 - ], - "id": "f60878f4-8cca-43d0-b8b3-0aa1a422237b", - "name": "Parse ingest" - }, - { - "parameters": { - "mode": "runOnceForEachItem", - "jsCode": "const d = $json;\nlet n;\nif (d.status === 'ok') {\n n = { title: `Ingest ${d.slug}: PR aperta`, priority: 'default', tags: 'inbox_tray',\n body: `\\u2705 ${d.slug}: PR aperta (lint ${d.lint_clean ? 'clean' : 'KO'}${d.conflict ? ', CONFLITTO' : ''})\\n\\n\\ud83d\\udd17 ${d.pr_url}` };\n} else if (d.status === 'pr_failed') {\n n = { title: `Ingest ${d.slug}: PR FALLITA`, priority: 'high', tags: 'warning',\n body: `\\u26a0\\ufe0f ${d.slug}: semantic/lint ok ma PR non aperta.\\n\\n${(d.detail || '').split('\\n')[0]}` };\n} else {\n n = { title: 'Ingest: ERRORE', priority: 'high', tags: 'rotating_light',\n body: `\\u274c ${d.reason || 'errore'}\\n\\n${(d.raw || '').slice(0,300)}` };\n}\nreturn n;" - }, - "type": "n8n-nodes-base.code", - "typeVersion": 2, - "position": [ - 2592, - 240 - ], - "id": "9018dff6-b314-4ca8-b8ff-fd5423818816", - "name": "Build ntfy" + 3232, + 1840 + ] }, { "parameters": { "method": "POST", - "url": "http://ntfy/homelab-genome", + "url": "=http://ntfy/{{ $json.topic }}", "authentication": "genericCredentialType", "genericAuthType": "httpBearerAuth", "sendHeaders": true, @@ -195,6 +271,18 @@ { "name": "Tags", "value": "={{ $json.tags }}" + }, + { + "name": "Click", + "value": "={{ $json.click }}" + }, + { + "name": "Actions", + "value": "={{ $json.actions }}" + }, + { + "name": "Markdown", + "value": "yes" } ] }, @@ -202,16 +290,18 @@ "contentType": "raw", "rawContentType": "Raw / Text", "body": "={{ $json.body }}", - "options": {} + "options": { + "timeout": 15000 + } }, + "id": "55cc42b2-3170-4884-bb5d-58e3af97bfea", + "name": "ntfy: send", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.4, "position": [ - 2800, - 240 + 3456, + 1840 ], - "id": "1f572cb3-741b-46bc-87fa-1e23ade5a9be", - "name": "ntfy: send notification", "credentials": { "httpHeaderAuth": { "id": "TBPXSWOF63k9mvm8", @@ -222,53 +312,6 @@ "name": "Bearer Auth account" } } - }, - { - "parameters": { - "conditions": { - "options": { - "caseSensitive": true, - "leftValue": "", - "typeValidation": "loose", - "version": 2 - }, - "conditions": [ - { - "id": "dd000000-0000-4000-8000-000000000001", - "leftValue": "={{ $json._kind }}", - "rightValue": "ingest", - "operator": { - "type": "string", - "operation": "equals" - } - } - ], - "combinator": "and" - }, - "options": {} - }, - "type": "n8n-nodes-base.if", - "typeVersion": 2.2, - "position": [ - 3168, - 240 - ], - "id": "d2d2e2b2-9bd7-446b-b6b8-0a865d49c601", - "name": "IF: nome valido" - }, - { - "parameters": { - "mode": "runOnceForEachItem", - "jsCode": "const d=$json;\nreturn {\n title: 'Ingest: nome file non valido',\n priority: 'high',\n tags: 'warning',\n body: `\\u26a0\\ufe0f \"${d.raw}\" ha spazi o caratteri non ammessi e non e' stato ingerito.\\n\\nRinominalo in: ${d.hint}`\n};" - }, - "type": "n8n-nodes-base.code", - "typeVersion": 2, - "position": [ - 3392, - 400 - ], - "id": "3b69aa97-170a-4666-8b0e-4b51b48b2817", - "name": "Build ntfy badname" } ], "pinData": {}, @@ -277,14 +320,14 @@ "main": [ [ { - "node": "IF: ref == develop", + "node": "Gate push", "type": "main", "index": 0 } ] ] }, - "IF: ref == develop": { + "Gate push": { "main": [ [ { @@ -299,14 +342,14 @@ "main": [ [ { - "node": "SSH: changed-raw", + "node": "SSH: pending-raw", "type": "main", "index": 0 } ] ] }, - "SSH: changed-raw": { + "SSH: pending-raw": { "main": [ [ { @@ -321,51 +364,18 @@ "main": [ [ { - "node": "IF: nome valido", + "node": "Nome valido?", "type": "main", "index": 0 } ] ] }, - "SSH: pi ingest": { + "Nome valido?": { "main": [ [ { - "node": "Parse ingest", - "type": "main", - "index": 0 - } - ] - ] - }, - "Parse ingest": { - "main": [ - [ - { - "node": "Build ntfy", - "type": "main", - "index": 0 - } - ] - ] - }, - "Build ntfy": { - "main": [ - [ - { - "node": "ntfy: send notification", - "type": "main", - "index": 0 - } - ] - ] - }, - "IF: nome valido": { - "main": [ - [ - { - "node": "SSH: pi ingest", + "node": "Run one ingest", "type": "main", "index": 0 } @@ -383,7 +393,7 @@ "main": [ [ { - "node": "ntfy: send notification", + "node": "ntfy: send", "type": "main", "index": 0 } @@ -394,9 +404,13 @@ "active": true, "settings": { "executionOrder": "v1", - "binaryMode": "separate" + "binaryMode": "separate", + "timeSavedMode": "fixed", + "errorWorkflow": "7Vws3gCX3QnjM3oD", + "callerPolicy": "workflowsFromSameOwner", + "availableInMCP": false }, - "versionId": "2115dd9f-e2b6-4acb-8de0-4a166eb9729a", + "versionId": "d58601e7-b752-4c9f-9438-d91be663c82e", "meta": { "instanceId": "96b2f0ec76a4400bbd481c617b24b3b87024cc7a913efacccaf9fc85722e7417" },