summary: Scrape.do — Bulk URL Scraper
description: >
  Scrapes a list of URLs through the Scrape.do API in parallel and returns
  structured results (status, title, h1, html length) for each one.
  Proxy rotation, anti-bot bypass, CAPTCHA solving and JS rendering are
  handled by Scrape.do — set `render` or `super` to true when a target needs them.

  Get a free token (1,000 requests/month): https://scrape.do

schema:
  $schema: 'https://json-schema.org/draft/2020-12/schema'
  type: object
  required:
    - token
    - urls
  order:
    - token
    - urls
    - render
    - super_proxy
  properties:
    token:
      type: string
      description: Your Scrape.do API token. Click the $ icon to store it as a Windmill secret variable instead of typing it inline.
      password: true
    urls:
      type: array
      description: Target URLs to scrape.
      items:
        type: string
      default:
        - https://books.toscrape.com/catalogue/page-1.html
        - https://books.toscrape.com/catalogue/page-2.html
        - https://books.toscrape.com/catalogue/page-3.html
    render:
      type: boolean
      description: Enable JavaScript rendering (headless browser). Costs extra credits — only for JS-heavy targets.
      default: false
    super_proxy:
      type: boolean
      description: Route through residential/mobile proxies. Costs extra credits — use for hard targets or persistent 502s.
      default: false

value:
  modules:
    - id: scrape_loop
      summary: Scrape each URL in parallel
      value:
        type: forloopflow
        parallel: true
        parallelism: 5
        skip_failures: true
        iterator:
          type: javascript
          expr: flow_input.urls
        modules:
          - id: scrape
            summary: Scrape.do request
            retry:
              constant:
                attempts: 2
                seconds: 3
            continue_on_error: true
            value:
              type: rawscript
              language: bun
              input_transforms:
                url:
                  type: javascript
                  expr: flow_input.iter.value
                token:
                  type: javascript
                  expr: flow_input.token
                render:
                  type: javascript
                  expr: flow_input.render
                super_proxy:
                  type: javascript
                  expr: flow_input.super_proxy
              content: |
                export async function main(
                  url: string,
                  token: string,
                  render = false,
                  super_proxy = false
                ) {
                  // Scrape.do responds ONLY at the root path "/" — never append /scrape.
                  const params = new URLSearchParams({ token, url });
                  if (render) params.set("render", "true");
                  if (super_proxy) params.set("super", "true");

                  // URLSearchParams handles URL-encoding of the target URL.
                  const res = await fetch(`https://api.scrape.do/?${params.toString()}`, {
                    signal: AbortSignal.timeout(120_000),
                  });

                  const body = await res.text();

                  if (res.status !== 200) {
                    // 502 ROTATION_FAILED is transient and NOT charged — the retry policy
                    // on this step handles it. Throwing lets the retry kick in.
                    throw new Error(`Scrape.do ${res.status}: ${body.slice(0, 300)}`);
                  }

                  const pick = (re: RegExp) => (body.match(re) || [])[1] || "";
                  const clean = (s: string) =>
                    s.replace(/<[^>]*>/g, "").replace(/\s+/g, " ").trim();

                  return {
                    url,
                    status: res.status,
                    title: clean(pick(/<title[^>]*>([\s\S]*?)<\/title>/i)),
                    h1: clean(pick(/<h1[^>]*>([\s\S]*?)<\/h1>/i)),
                    html_length: body.length,
                    scraped_at: new Date().toISOString(),
                  };
                }

    - id: summarize
      summary: Collect results
      value:
        type: rawscript
        language: bun
        input_transforms:
          results:
            type: javascript
            expr: results.scrape_loop
        content: |
          export async function main(results: any[]) {
            // Failed iterations come back as error objects because the loop
            // runs with skip_failures: true — split them out cleanly.
            const ok = results.filter((r) => r && r.status === 200);
            const failed = results.filter((r) => !r || r.status !== 200);

            return {
              total: results.length,
              success: ok.length,
              failed: failed.length,
              rows: ok,
              errors: failed,
            };
          }