logo

Browser Interactions

Scrape.do supports basic JavaScript instructions to interact with the web page you want to scrape. You can create a list of instructions and send it to the system. The system will execute the instructions in the order you send them and return the result to you.

You should URL-encode the playWithBrowser query parameter value. You can see more details about URL encoding here.

Example of a script with multiple instructions:

[
    { "Action": "Click", "Selector": "#button_id" },
    { "Action": "Wait", "Timeout": 5000 }
]
curl --location --request GET 'http://api.scrape.do/?render=true&playWithBrowser=%5B%7B%22Action%22%3A%22Click%22%2C%22Selector%22%3A%22%23html-page%22%7D%5D&token=YOUR_TOKEN&url=https://httpbin.co/' \
import requests
import urllib.parse

jsonData = '[{"Action": "Click","Selector":"#html-page"}]'
encodedJsonData = urllib.parse.quote_plus(jsonData)
token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote_plus("https://httpbin.co/")
render = "true"
url = f"http://api.scrape.do/?token={token}&url={targetUrl}&render={render}&playWithBrowser={encodedJsonData}"
response = requests.request("GET", url)
print(response.text)
const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/");
const jsonData = '[{"Action":"Click","Selector":"#html-page"}]';
const encodedJsonData = encodeURIComponent(jsonData);
const render = "true";
const config = {
    'method': 'GET',
    'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&render=${render}&playWithBrowser=${encodedJsonData}`
};
axios(config)
    .then(function (response) {
        console.log(response.data);
    })
    .catch(function (error) {
        console.log(error);
    });
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	token := "YOUR_TOKEN"
	encoded_url := url.QueryEscape("https://httpbin.co/")
	jsonData := `[{"Action":"Click","Selector":"#html-page"}]`
	encodedJsonData := url.QueryEscape(jsonData)
	url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&render=true&playWithBrowser=%s", token, encoded_url, encodedJsonData)
	method := "GET"
	client := &http.Client{}
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		fmt.Println(err)
		return
	}
	res, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))
}
require "uri"
require "net/http"
require 'cgi'
require "base64"
require 'json'

jsonData = [
  {
    "Action": "Click",
    "Selector": "#html-page"
  }
]
encodedJsonData = CGI.escape(JSON.dump(jsonData))
str = CGI.escape "https://httpbin.co/"
url = URI("https://api.scrape.do/?url=" + str + "&token=YOUR_TOKEN&render=true&playWithBrowser="+ encodedJsonData+"")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
response = https.request(request)
puts response.read_body
String jsonData = "[{\"Action\": \"Click\",\"Selector\":\"#html-page\"}]";
String encodedJsonData = URLEncoder.encode(jsonData, StandardCharsets.UTF_8);
String token = "YOUR_TOKEN";
String targetUrl = URLEncoder.encode("https://httpbin.co/", StandardCharsets.UTF_8);
String render = "true";
String url = String.format("http://api.scrape.do/?token=%s&url=%s&render=%s&playWithBrowser=%s", token, targetUrl, render, encodedJsonData);
HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
        .GET()
        .uri(URI.create(url))
        .build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());
string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/");
var myData = new List<Object>()
{
    new
    {
        Action = "Click",
        Selector = "#html-page"
    }
};
 string jsonData = JsonConvert.SerializeObject(myData);
 string encodedResult = WebUtility.UrlEncode(jsonData);
 var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&playWithBrowser={encodedResult}&render=true";        
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);
<?php
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_HEADER, false);
$array = [
    [
        'Action' => 'Click',
        'Selector'  => '#html-page'
    ]
];
$encodedJsonData = urlencode(json_encode($array));
$data = [
   "url" => "https://httpbin.co/",
   "token" => "YOUR_TOKEN",
   "render"=> "true",
   "PlayWithBrowser" => urldecode($encodedJsonData)
];
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($curl, CURLOPT_URL, "https://api.scrape.do/?" . http_build_query($data));
curl_setopt($curl, CURLOPT_HTTPHEADER, array(
       "Accept: */*",
));
$response = curl_exec($curl);
curl_close($curl);
echo $response;

You can add multiple instructions to the script. Below are examples of all the different instructions you can use.

When you also pass returnJSON=true, the response includes an actionResults array reporting success, and any error, for each action in your script, in order. A failed selector lookup is reported there while the request itself still returns 200.

Actions

Click

To click a button, you will need to use this feature. Use it with the CSS selector of the button you want to click. The scraper will click on the #buttonId button and then return the HTML of the target page.

[{ "Action": "Click", "Selector": "#button_id"}]

Wait

Use the wait command with the time in ms you want to "wait" for when you need to wait for a fixed amount of time.

[{ "Action": "Wait", "Timeout": 1000 }]

Wait Selector

If there is no expected content as a result of a successful return, you can wait for the element by specifying the desired time in ms with the "wait-selector" property.

[{ "Action": "WaitSelector", "WaitSelector": "#btn", "Timeout": 1000 }]

Scroll Horizontally

Use the "scroll-x" attribute with the number of pixels you want to scroll to scroll the target website page horizontally.

[{ "Action": "ScrollX", "Value": 100 }]

Scroll Vertically

To scroll the target website page vertically, use the "scroll-y" attribute with the number of pixels you want to scroll.

[{ "Action": "ScrollY", "Value": 100 }]

Scroll To

With the "scroll-to" property, you can scroll for x - y on a per-pixel basis.

[{ "Action": "ScrollTo", "Selector": "#btn" }]

Fill

Use the "fill" attribute to fill an entry on the target website page. Use it with the CSS selector of the input you want to fill and the value you want to fill it with.

[{ "Action": "Fill", "Selector": "#input", "Value": "test" }]

Execute

JavaScript code is executed. If you need to run custom JavaScript, you should use the "execute" property.

 [{ "Action": "Execute", "Execute": "window.document.cookie" }]

Screenshot

The returnJSON=true parameter needs to be used.


[{ "Action": "ScreenShot" }]

[{ "Action": "ScreenShot", "fullScreenShot": "true" }]

[{ "Action": "ScreenShot", "particularScreenShot": "#elementSelector" }]

WaitForRequestCompletion

"WaitForRequestCompletion" waits for any network request matching the given urlPattern to complete or until the specified timeout is reached. Useful for waiting on dynamically loaded content such as APIs, images, scripts, or other resources.

[{ "Action": "WaitForRequestCompletion", "UrlPattern": "*example.com/image*", "Timeout": 10000 }]

RealClick

Selects a random coordinate inside the clickable element and clicks it with operating system specific human behavior, instead of dispatching a synthetic click at the element center. Use it when a plain "Click" gets detected or ignored by the target site.

[{ "Action": "RealClick", "Selector": "#buy-button" }]

If the click triggers a page navigation, use ClickAndWait instead so the navigation is tracked and completed before the result is returned.

RealAction

Simulates human behavior on the page for the given time period, specific to the operating system. On desktop it performs mouse movements and scroll wheel scrolling. On mobile it interacts with the page through touch events. Useful for warming up sessions on sites that score user behavior.

[{ "Action": "RealAction", "Timeout": 10000 }]

Focus

Triggers the focus event on an element, the same as a user tabbing or clicking into it. Useful before typing into inputs that only initialize on focus.

[{ "Action": "Focus", "Selector": "#search-input" }]

Hover

Triggers the hover event on an element. Useful for opening hover-only menus or lazy-loading content that appears on mouseover.

[{ "Action": "Hover", "Selector": ".dropdown-menu" }]

Select

Changes the value of a <select> element and fires its change event.

[{ "Action": "Select", "Selector": "#country", "Value": "US" }]

WaitForFunction

Waits until a JavaScript expression on the page returns a truthy value. For example, to wait until window.turnstile is available, pass window.turnstile as the function. Timeout is optional and given in ms.

[{ "Action": "WaitForFunction", "Function": "window.turnstile", "Timeout": 10000 }]

ClickAndWait

Clicks an element and waits for the resulting navigation to complete according to the WaitUntil type you specify, such as domcontentloaded or load. Use this instead of "Click" or "RealClick" whenever the click navigates to another page. Timeout is optional and given in ms.

[{ "Action": "ClickAndWait", "Selector": "a.next-page", "WaitUntil": "domcontentloaded", "Timeout": 10000 }]

Tap

Handles the touch event on mobile devices. Click events do not work on our Android-based mobile browser infrastructure, so use Tap for taps when scraping with mobile emulation.

[{ "Action": "Tap", "Selector": "#load-more" }]

On this page