Browser Interactions
Scrape.do supports basic JavaScript instructions to interact with the web page you want to scrape. You can create a list of instructions and send it to the system. The system will execute the instructions in the order you send them and return the result to you.
You should URL-encode the playWithBrowser query parameter value. You can see more details about URL encoding here.
Example of a script with multiple instructions:
[
{ "Action": "Click", "Selector": "#button_id" },
{ "Action": "Wait", "Timeout": 5000 }
]curl --location --request GET 'http://api.scrape.do/?render=true&playWithBrowser=%5B%7B%22Action%22%3A%22Click%22%2C%22Selector%22%3A%22%23html-page%22%7D%5D&token=YOUR_TOKEN&url=https://httpbin.co/' \import requests
import urllib.parse
jsonData = '[{"Action": "Click","Selector":"#html-page"}]'
encodedJsonData = urllib.parse.quote_plus(jsonData)
token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote_plus("https://httpbin.co/")
render = "true"
url = f"http://api.scrape.do/?token={token}&url={targetUrl}&render={render}&playWithBrowser={encodedJsonData}"
response = requests.request("GET", url)
print(response.text)const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/");
const jsonData = '[{"Action":"Click","Selector":"#html-page"}]';
const encodedJsonData = encodeURIComponent(jsonData);
const render = "true";
const config = {
'method': 'GET',
'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&render=${render}&playWithBrowser=${encodedJsonData}`
};
axios(config)
.then(function (response) {
console.log(response.data);
})
.catch(function (error) {
console.log(error);
});package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
token := "YOUR_TOKEN"
encoded_url := url.QueryEscape("https://httpbin.co/")
jsonData := `[{"Action":"Click","Selector":"#html-page"}]`
encodedJsonData := url.QueryEscape(jsonData)
url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&render=true&playWithBrowser=%s", token, encoded_url, encodedJsonData)
method := "GET"
client := &http.Client{}
req, err := http.NewRequest(method, url, nil)
if err != nil {
fmt.Println(err)
return
}
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}require "uri"
require "net/http"
require 'cgi'
require "base64"
require 'json'
jsonData = [
{
"Action": "Click",
"Selector": "#html-page"
}
]
encodedJsonData = CGI.escape(JSON.dump(jsonData))
str = CGI.escape "https://httpbin.co/"
url = URI("https://api.scrape.do/?url=" + str + "&token=YOUR_TOKEN&render=true&playWithBrowser="+ encodedJsonData+"")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
response = https.request(request)
puts response.read_bodyString jsonData = "[{\"Action\": \"Click\",\"Selector\":\"#html-page\"}]";
String encodedJsonData = URLEncoder.encode(jsonData, StandardCharsets.UTF_8);
String token = "YOUR_TOKEN";
String targetUrl = URLEncoder.encode("https://httpbin.co/", StandardCharsets.UTF_8);
String render = "true";
String url = String.format("http://api.scrape.do/?token=%s&url=%s&render=%s&playWithBrowser=%s", token, targetUrl, render, encodedJsonData);
HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
.GET()
.uri(URI.create(url))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/");
var myData = new List<Object>()
{
new
{
Action = "Click",
Selector = "#html-page"
}
};
string jsonData = JsonConvert.SerializeObject(myData);
string encodedResult = WebUtility.UrlEncode(jsonData);
var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&playWithBrowser={encodedResult}&render=true";
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);You can add multiple instructions to the script. Below are examples of all the different instructions you can use.
Actions
Click
To click a button, you will need to use this feature. Use it with the CSS selector of the button you want to click. The scraper will click on the #buttonId button and then return the HTML of the target page.
[{ "Action": "Click", "Selector": "#button_id"}]Wait
Use the wait command with the time in ms you want to "wait" for when you need to wait for a fixed amount of time.
[{ "Action": "Wait", "Timeout": 1000 }]Wait Selector
If there is no expected content as a result of a successful return, you can wait for the element by specifying the desired time in ms with the "wait-selector" property.
[{ "Action": "WaitSelector", "WaitSelector": "#btn", "Timeout": 1000 }]Scroll Horizontally
Use the "scroll-x" attribute with the number of pixels you want to scroll to scroll the target website page horizontally.
[{ "Action": "ScrollX", "Value": 100 }]Scroll Vertically
To scroll the target website page vertically, use the "scroll-y" attribute with the number of pixels you want to scroll.
[{ "Action": "ScrollY", "Value": 100 }]Scroll To
With the "scroll-to" property, you can scroll for x - y on a per-pixel basis.
[{ "Action": "ScrollTo", "Selector": "#btn" }]Fill
Use the "fill" attribute to fill an entry on the target website page. Use it with the CSS selector of the input you want to fill and the value you want to fill it with.
[{ "Action": "Fill", "Selector": "#input", "Value": "test" }]Execute
JavaScript code is executed. If you need to run custom JavaScript, you should use the "execute" property.
[{ "Action": "Execute", "Execute": "window.document.cookie" }]Screenshot
The returnJSON=true parameter needs to be used.
[{ "Action": "ScreenShot" }]
[{ "Action": "ScreenShot", "fullScreenShot": "true" }]
[{ "Action": "ScreenShot", "particularScreenShot": "#elementSelector" }]WaitForRequestCompletion
"WaitForRequestCompletion" waits for any network request matching the given urlPattern to complete or until the specified timeout is reached. Useful for waiting on dynamically loaded content such as APIs, images, scripts, or other resources.
[{ "Action": "WaitForRequestCompletion", "UrlPattern": "*example.com/image*", "Timeout": 10000 }]
