version: 6.48.0

type: DeclarativeSource

description: >-
  Scrape.do source connector. Fetches a user-supplied list of URLs through the
  Scrape.do web scraping API and emits one record per URL. Each request runs in
  a headless browser with returnJSON, so every page arrives as a JSON record
  with its rendered HTML in the content field, ready to parse in your
  warehouse.

check:
  type: CheckStream
  stream_names:
    - scraped_pages

definitions:
  base_requester:
    type: HttpRequester
    url_base: https://api.scrape.do
    http_method: GET
    authenticator:
      type: NoAuth
    error_handler:
      type: DefaultErrorHandler
      response_filters:
        - type: HttpResponseFilter
          http_codes: [401, 403]
          action: FAIL
          error_message: >-
            Scrape.do refused the request. Check the API token in your connector
            settings, and the credits and subscription status in your Scrape.do
            dashboard.
        - type: HttpResponseFilter
          http_codes: [429]
          action: RATE_LIMITED
          error_message: Concurrency limit reached for this Scrape.do plan.
        - type: HttpResponseFilter
          http_codes: [502, 503]
          action: RETRY
          error_message: Proxy could not reach the target. Retrying.
      backoff_strategies:
        - type: ExponentialBackoffStrategy
          factor: 2

  scraped_pages_stream:
    type: DeclarativeStream
    name: scraped_pages
    primary_key:
      - source_url
    retriever:
      type: SimpleRetriever
      requester:
        $ref: "#/definitions/base_requester"
        path: "/"
        request_parameters:
          token: "{{ config['api_token'] }}"
          url: "{{ stream_partition.target_url }}"
          render: "true"
          returnJSON: "true"
          super: "{{ config['super_proxy'] | lower }}"
          geoCode: "{{ config['geo_code'] }}"
      record_selector:
        type: RecordSelector
        extractor:
          type: DpathExtractor
          field_path: []
      partition_router:
        type: ListPartitionRouter
        cursor_field: target_url
        values: "{{ config['urls'] }}"
    transformations:
      - type: AddFields
        fields:
          - path:
              - source_url
            value: "{{ stream_partition.target_url }}"
          - path:
              - scraped_at
            value: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}"
      - type: RemoveFields
        field_pointers:
          - - networkRequests
          - - websocketRequests
          - - actionResults
          - - screenShots
    schema_loader:
      type: InlineSchemaLoader
      schema:
        $ref: "#/schemas/scraped_pages"

streams:
  - $ref: "#/definitions/scraped_pages_stream"

spec:
  type: Spec
  connection_specification:
    type: object
    $schema: http://json-schema.org/draft-07/schema#
    title: Scrape.do Source Spec
    required:
      - api_token
      - urls
    additionalProperties: true
    properties:
      api_token:
        type: string
        title: API Token
        description: >-
          Your Scrape.do API token. Find it at https://dashboard.scrape.do
        airbyte_secret: true
        order: 0
      urls:
        type: array
        title: URLs to Scrape
        description: The list of page addresses to fetch. One record is emitted per URL.
        items:
          type: string
        examples:
          - - https://us.amazon.com/dp/B0BLRJ4R8F
            - https://us.amazon.com/dp/B0CHX1W1XY
        order: 1
      super_proxy:
        type: boolean
        title: Use Residential Proxies
        description: >-
          Turn on for heavily protected sites. Uses residential and mobile IPs
          instead of datacenter IPs. Costs more credits per request.
        default: false
        order: 2
      geo_code:
        type: string
        title: Proxy Country
        description: Two letter country code for the proxy location.
        default: us
        examples:
          - us
          - gb
          - de
        order: 3

schemas:
  scraped_pages:
    type: object
    $schema: http://json-schema.org/draft-07/schema#
    additionalProperties: true
    properties:
      source_url:
        type:
          - "null"
          - string
        description: The URL this record was scraped from.
      scraped_at:
        type:
          - "null"
          - string
        format: date-time
        description: When the request was made.
      statusCode:
        type:
          - "null"
          - integer
        description: The status code the target page returned to Scrape.do.
      content:
        type:
          - "null"
          - string
        description: The rendered HTML of the page.

metadata:
  autoImportSchema:
    scraped_pages: false
  testedStreams:
    scraped_pages:
      hasRecords: false
      streamHash: null
      hasResponse: false
      primaryKeysAreUnique: false
      primaryKeysArePresent: false
      responsesAreSuccessful: false
  assist: {}