{
  "exportVersion": 1,
  "workflow": {
    "name": "scrapedo-bulk-scrape",
    "description": "Bulk scrape a list of URLs with Scrape.do and write the results back to Retool DB.",
    "isEnabled": false,
    "crontab": "0 * * * *",
    "timezone": "Europe/Istanbul",
    "triggerWebhooks": [],
    "configVars": [
      {
        "name": "SCRAPE_DO_TOKEN",
        "description": "Scrape.do API token. Set this in Settings > Configuration variables before running.",
        "value": ""
      }
    ],
    "blocks": [
      {
        "id": "startTrigger",
        "pluginType": "StartTrigger",
        "position": { "x": 0, "y": 0 },
        "template": {
          "triggerType": "manual"
        },
        "dependencies": []
      },
      {
        "id": "getUrls",
        "pluginType": "DatasourceQuery",
        "resourceName": "retool_db",
        "position": { "x": 0, "y": 180 },
        "template": {
          "queryType": "sql",
          "query": "select id, url from scrape_targets where status = 'pending' limit 500",
          "runWhenModelUpdates": false
        },
        "dependencies": ["startTrigger"]
      },
      {
        "id": "scrapeLoop",
        "pluginType": "LoopBlock",
        "position": { "x": 0, "y": 360 },
        "template": {
          "loopMode": "gui",
          "iterable": "{{ getUrls.data }}",
          "executionMode": "batch",
          "batchSize": 5,
          "iterationDelayMs": 250,
          "continueOnFailure": true,
          "innerBlock": {
            "id": "scrapeOne",
            "pluginType": "RESTQuery",
            "resourceName": "Scrape.do",
            "template": {
              "method": "GET",
              "path": "",
              "queryParams": [
                ["url", "{{ encodeURIComponent(value.url) }}"],
                ["output", "markdown"],
                ["render", "false"]
              ],
              "headers": [],
              "bodyType": "none",
              "body": []
            }
          }
        },
        "dependencies": ["getUrls"]
      },
      {
        "id": "normalize",
        "pluginType": "JavascriptQuery",
        "position": { "x": 0, "y": 540 },
        "template": {
          "code": "return scrapeLoop.data.map((res, i) => {\n  const target = getUrls.data[i];\n  const ok = res !== null && res !== undefined && res !== \"\";\n  return {\n    id: target.id,\n    url: target.url,\n    content: ok ? res : null,\n    status: ok ? \"ok\" : \"failed\",\n    scraped_at: new Date().toISOString()\n  };\n});"
        },
        "dependencies": ["scrapeLoop"]
      },
      {
        "id": "save",
        "pluginType": "DatasourceQuery",
        "resourceName": "retool_db",
        "position": { "x": 0, "y": 720 },
        "template": {
          "queryType": "bulkUpdate",
          "table": "scrape_targets",
          "primaryKey": "id",
          "arrayValue": "{{ normalize.data }}"
        },
        "dependencies": ["normalize"]
      },
      {
        "id": "returnValue",
        "pluginType": "ResponseBlock",
        "position": { "x": 0, "y": 900 },
        "template": {
          "body": "{{ { total: normalize.data.length, ok: normalize.data.filter(r => r.status === 'ok').length, failed: normalize.data.filter(r => r.status === 'failed').length } }}"
        },
        "dependencies": ["save"]
      }
    ]
  },
  "requiredResources": [
    {
      "name": "Scrape.do",
      "type": "restapi",
      "baseURL": "https://api.scrape.do/",
      "authentication": "none",
      "urlParameters": [
        { "key": "token", "value": "{{ configVars.SCRAPE_DO_TOKEN }}" }
      ]
    },
    {
      "name": "retool_db",
      "type": "retoolDatabase",
      "schema": "create table scrape_targets (id serial primary key, url text not null, status text not null default 'pending', content text, scraped_at timestamptz);"
    }
  ]
}