app.bettersight.io/n8n/workflows/daily_scrape_cron.json

{
  "name": "Bettersight — Daily Scrape Cron",
  "nodes": [
    {
      "parameters": {
        "rule": {
          "interval": [{ "field": "cronExpression", "expression": "0 6 * * *" }]
        }
      },
      "id": "schedule-trigger",
      "name": "Schedule Trigger (06:00 daily)",
      "type": "n8n-nodes-base.scheduleTrigger",
      "typeVersion": 1.2,
      "position": [0, 0]
    },
    {
      "parameters": {
        "method": "GET",
        "url": "={{ $env.POCKETBASE_URL }}/api/collections/tenants/records",
        "sendQuery": true,
        "queryParameters": {
          "parameters": [
            { "name": "filter", "value": "status = 'active' || status = 'trial'" },
            { "name": "perPage", "value": "200" }
          ]
        },
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth"
      },
      "id": "list-active-tenants",
      "name": "List active tenants (PocketBase)",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [220, 0],
      "credentials": {
        "httpHeaderAuth": { "id": "pocketbase-admin-auth", "name": "PocketBase Admin Auth" }
      },
      "notes": "Header credential should carry the PocketBase admin session token — see repositories/pocketbase_client.py's own auth flow for the equivalent Python-side pattern."
    },
    {
      "parameters": { "fieldToSplitOut": "items" },
      "id": "split-tenants",
      "name": "Split into items",
      "type": "n8n-nodes-base.splitOut",
      "typeVersion": 1,
      "position": [440, 0]
    },
    {
      "parameters": {
        "batchSize": 1
      },
      "id": "loop-tenants",
      "name": "Loop Over Tenants",
      "type": "n8n-nodes-base.splitInBatches",
      "typeVersion": 3,
      "position": [660, 0]
    },
    {
      "parameters": {
        "method": "GET",
        "url": "={{ $env.POCKETBASE_URL }}/api/collections/competitors/records",
        "sendQuery": true,
        "queryParameters": {
          "parameters": [
            {
              "name": "filter",
              "value": "={{ 'tenant_id = \"' + $json.id + '\" && active = true && (change_detected = true || last_scraped = \"\" || last_scraped < \"' + $now.minus(24, 'hours').toISO() + '\")' }}"
            },
            { "name": "perPage", "value": "200" }
          ]
        },
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth"
      },
      "id": "list-stale-competitors",
      "name": "List stale/changed competitors",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [880, -80],
      "credentials": {
        "httpHeaderAuth": { "id": "pocketbase-admin-auth", "name": "PocketBase Admin Auth" }
      }
    },
    {
      "parameters": {
        "conditions": {
          "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose" },
          "conditions": [
            {
              "leftValue": "={{ $json.items.length }}",
              "rightValue": 0,
              "operator": { "type": "number", "operation": "gt" }
            }
          ],
          "combinator": "and"
        }
      },
      "id": "has-stale-competitors",
      "name": "Any stale competitors?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [1100, -80]
    },
    {
      "parameters": {
        "method": "POST",
        "url": "={{ $env.BETTERSIGHT_API_BASE_URL }}/internal/scrape/{{ $('Loop Over Tenants').item.json.id }}",
        "sendHeaders": true,
        "headerParameters": {
          "parameters": [{ "name": "X-API-Key", "value": "={{ $env.API_SECRET_KEY }}" }]
        },
        "sendBody": true,
        "bodyParameters": {
          "parameters": [
            { "name": "competitors", "value": "={{ $json.items.map(c => ({ id: c.id, name: c.name })) }}" }
          ]
        }
      },
      "id": "trigger-scrape",
      "name": "POST /internal/scrape/{tenant_id}",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [1320, -80]
    }
  ],
  "connections": {
    "Schedule Trigger (06:00 daily)": {
      "main": [[{ "node": "List active tenants (PocketBase)", "type": "main", "index": 0 }]]
    },
    "List active tenants (PocketBase)": {
      "main": [[{ "node": "Split into items", "type": "main", "index": 0 }]]
    },
    "Split into items": {
      "main": [[{ "node": "Loop Over Tenants", "type": "main", "index": 0 }]]
    },
    "Loop Over Tenants": {
      "main": [
        [],
        [{ "node": "List stale/changed competitors", "type": "main", "index": 0 }]
      ]
    },
    "List stale/changed competitors": {
      "main": [[{ "node": "Any stale competitors?", "type": "main", "index": 0 }]]
    },
    "Any stale competitors?": {
      "main": [
        [{ "node": "POST /internal/scrape/{tenant_id}", "type": "main", "index": 0 }],
        []
      ]
    },
    "POST /internal/scrape/{tenant_id}": {
      "main": [[{ "node": "Loop Over Tenants", "type": "main", "index": 0 }]]
    }
  },
  "active": false,
  "settings": { "executionOrder": "v1" },
  "meta": {
    "description": "CLAUDE.md §15 daily scrape cron — queries PocketBase directly for each active tenant's stale/changed competitors (last_scraped > 24hrs OR change_detected = true), then triggers a scrape job per tenant via the existing Flask /internal/scrape/<tenant_id> route. Not yet imported/run against a live n8n instance — verify the PocketBase filter syntax and credential wiring before enabling."
  }
}