Browser Interactions
Scrape.do supports basic JavaScript instructions to interact with the web page you want to scrape. You can create a list of instructions and send it to the system. The system will execute the instructions in the order you send them and return the result to you.
You should URL-encode the playWithBrowser query parameter value. You can see more details about URL encoding here.
Example of a script with multiple instructions:
[
{ "Action": "Click", "Selector": "#button_id" },
{ "Action": "Wait", "Timeout": 5000 }
]curl --location --request GET 'http://api.scrape.do/?render=true&playWithBrowser=%5B%7B%22Action%22%3A%22Click%22%2C%22Selector%22%3A%22%23html-page%22%7D%5D&token=YOUR_TOKEN&url=https://httpbin.co/' \import requests
import urllib.parse
jsonData = '[{"Action": "Click","Selector":"#html-page"}]'
encodedJsonData = urllib.parse.quote_plus(jsonData)
token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote_plus("https://httpbin.co/")
render = "true"
url = f"http://api.scrape.do/?token={token}&url={targetUrl}&render={render}&playWithBrowser={encodedJsonData}"
response = requests.request("GET", url)
print(response.text)const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/");
const jsonData = '[{"Action":"Click","Selector":"#html-page"}]';
const encodedJsonData = encodeURIComponent(jsonData);
const render = "true";
const config = {
'method': 'GET',
'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&render=${render}&playWithBrowser=${encodedJsonData}`
};
axios(config)
.then(function (response) {
console.log(response.data);
})
.catch(function (error) {
console.log(error);
});package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
token := "YOUR_TOKEN"
encoded_url := url.QueryEscape("https://httpbin.co/")
jsonData := `[{"Action":"Click","Selector":"#html-page"}]`
encodedJsonData := url.QueryEscape(jsonData)
url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&render=true&playWithBrowser=%s", token, encoded_url, encodedJsonData)
method := "GET"
client := &http.Client{}
req, err := http.NewRequest(method, url, nil)
if err != nil {
fmt.Println(err)
return
}
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}require "uri"
require "net/http"
require 'cgi'
require "base64"
require 'json'
jsonData = [
{
"Action": "Click",
"Selector": "#html-page"
}
]
encodedJsonData = CGI.escape(JSON.dump(jsonData))
str = CGI.escape "https://httpbin.co/"
url = URI("https://api.scrape.do/?url=" + str + "&token=YOUR_TOKEN&render=true&playWithBrowser="+ encodedJsonData+"")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
response = https.request(request)
puts response.read_bodyString jsonData = "[{\"Action\": \"Click\",\"Selector\":\"#html-page\"}]";
String encodedJsonData = URLEncoder.encode(jsonData, StandardCharsets.UTF_8);
String token = "YOUR_TOKEN";
String targetUrl = URLEncoder.encode("https://httpbin.co/", StandardCharsets.UTF_8);
String render = "true";
String url = String.format("http://api.scrape.do/?token=%s&url=%s&render=%s&playWithBrowser=%s", token, targetUrl, render, encodedJsonData);
HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
.GET()
.uri(URI.create(url))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/");
var myData = new List<Object>()
{
new
{
Action = "Click",
Selector = "#html-page"
}
};
string jsonData = JsonConvert.SerializeObject(myData);
string encodedResult = WebUtility.UrlEncode(jsonData);
var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&playWithBrowser={encodedResult}&render=true";
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);<?php
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_HEADER, false);
$array = [
[
'Action' => 'Click',
'Selector' => '#html-page'
]
];
$encodedJsonData = urlencode(json_encode($array));
$data = [
"url" => "https://httpbin.co/",
"token" => "YOUR_TOKEN",
"render"=> "true",
"PlayWithBrowser" => urldecode($encodedJsonData)
];
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($curl, CURLOPT_URL, "https://api.scrape.do/?" . http_build_query($data));
curl_setopt($curl, CURLOPT_HTTPHEADER, array(
"Accept: */*",
));
$response = curl_exec($curl);
curl_close($curl);
echo $response;You can add multiple instructions to the script. Below are examples of all the different instructions you can use.
When you also pass returnJSON=true, the response includes an actionResults array reporting success, and any error, for each action in your script, in order. A failed selector lookup is reported there while the request itself still returns 200.
Actions
Click
To click a button, you will need to use this feature. Use it with the CSS selector of the button you want to click. The scraper will click on the #buttonId button and then return the HTML of the target page.
[{ "Action": "Click", "Selector": "#button_id"}]Wait
Use the wait command with the time in ms you want to "wait" for when you need to wait for a fixed amount of time.
[{ "Action": "Wait", "Timeout": 1000 }]Wait Selector
If there is no expected content as a result of a successful return, you can wait for the element by specifying the desired time in ms with the "wait-selector" property.
[{ "Action": "WaitSelector", "WaitSelector": "#btn", "Timeout": 1000 }]Scroll Horizontally
Use the "scroll-x" attribute with the number of pixels you want to scroll to scroll the target website page horizontally.
[{ "Action": "ScrollX", "Value": 100 }]Scroll Vertically
To scroll the target website page vertically, use the "scroll-y" attribute with the number of pixels you want to scroll.
[{ "Action": "ScrollY", "Value": 100 }]Scroll To
With the "scroll-to" property, you can scroll for x - y on a per-pixel basis.
[{ "Action": "ScrollTo", "Selector": "#btn" }]Fill
Use the "fill" attribute to fill an entry on the target website page. Use it with the CSS selector of the input you want to fill and the value you want to fill it with.
[{ "Action": "Fill", "Selector": "#input", "Value": "test" }]Execute
JavaScript code is executed. If you need to run custom JavaScript, you should use the "execute" property.
[{ "Action": "Execute", "Execute": "window.document.cookie" }]Screenshot
The returnJSON=true parameter needs to be used.
[{ "Action": "ScreenShot" }]
[{ "Action": "ScreenShot", "fullScreenShot": "true" }]
[{ "Action": "ScreenShot", "particularScreenShot": "#elementSelector" }]WaitForRequestCompletion
"WaitForRequestCompletion" waits for any network request matching the given urlPattern to complete or until the specified timeout is reached. Useful for waiting on dynamically loaded content such as APIs, images, scripts, or other resources.
[{ "Action": "WaitForRequestCompletion", "UrlPattern": "*example.com/image*", "Timeout": 10000 }]RealClick
Selects a random coordinate inside the clickable element and clicks it with operating system specific human behavior, instead of dispatching a synthetic click at the element center. Use it when a plain "Click" gets detected or ignored by the target site.
[{ "Action": "RealClick", "Selector": "#buy-button" }]If the click triggers a page navigation, use ClickAndWait instead so the navigation is tracked and completed before the result is returned.
RealAction
Simulates human behavior on the page for the given time period, specific to the operating system. On desktop it performs mouse movements and scroll wheel scrolling. On mobile it interacts with the page through touch events. Useful for warming up sessions on sites that score user behavior.
[{ "Action": "RealAction", "Timeout": 10000 }]Focus
Triggers the focus event on an element, the same as a user tabbing or clicking into it. Useful before typing into inputs that only initialize on focus.
[{ "Action": "Focus", "Selector": "#search-input" }]Hover
Triggers the hover event on an element. Useful for opening hover-only menus or lazy-loading content that appears on mouseover.
[{ "Action": "Hover", "Selector": ".dropdown-menu" }]Select
Changes the value of a <select> element and fires its change event.
[{ "Action": "Select", "Selector": "#country", "Value": "US" }]WaitForFunction
Waits until a JavaScript expression on the page returns a truthy value. For example, to wait until window.turnstile is available, pass window.turnstile as the function. Timeout is optional and given in ms.
[{ "Action": "WaitForFunction", "Function": "window.turnstile", "Timeout": 10000 }]ClickAndWait
Clicks an element and waits for the resulting navigation to complete according to the WaitUntil type you specify, such as domcontentloaded or load. Use this instead of "Click" or "RealClick" whenever the click navigates to another page. Timeout is optional and given in ms.
[{ "Action": "ClickAndWait", "Selector": "a.next-page", "WaitUntil": "domcontentloaded", "Timeout": 10000 }]Tap
Handles the touch event on mobile devices. Click events do not work on our Android-based mobile browser infrastructure, so use Tap for taps when scraping with mobile emulation.
[{ "Action": "Tap", "Selector": "#load-more" }]
