Custom Headers
Take full control of request headers
You can interfere with all headers sent to the target website via our service. Scrape.do sends User-Agent, Accept, Cookies, etc. from you. It takes all header parameters and forwards them. You must set customHeaders=True to use the feature.
curl --location --request GET 'https://api.scrape.do/?token=YOUR_TOKEN&url=https://httpbin.co/headers&customHeaders=True'
--header 'Test-Header-Key: TestValue'import requests
import urllib.parse
token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote("https://httpbin.co/headers")
customHeaders = "true"
payload = {}
url = "http://api.scrape.do/?token={}&url={}&customHeaders={}".format(token, targetUrl, customHeaders)
headers = {
'Test-Header-Key': 'TestValue'
}
response = requests.request("GET", url, headers=headers, data=payload)
print(response.text)const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/headers");
const config = {
'method': 'GET',
'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&customHeaders=true`,
'headers': {
'Test-Header-Key': 'TestValue',
}
};
axios(config)
.then(function (response) {
console.log(response.data);
})
.catch(function (error) {
console.log(error);
});package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
token := "YOUR_TOKEN"
encoded_url := url.QueryEscape("https://httpbin.co/headers")
url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&customHeaders=true", token, encoded_url)
method := "GET"
client := &http.Client{}
req, err := http.NewRequest(method, url, nil)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("Test-Header-Key", "TestValue")
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}require "uri"
require "net/http"
require 'cgi'
str = CGI.escape "https://httpbin.co/headers"
url = URI("https://api.scrape.do/?url=" + str + "&token=YOUR_TOKEN&customHeaders=true")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
request['Test-Header-Key'] = 'TestValue'
response = https.request(request)
puts response.read_bodyOkHttpClient client = new OkHttpClient().newBuilder()
.build();
MediaType mediaType = MediaType.parse("text/plain");
RequestBody body = RequestBody.create(mediaType, "");
String encoded_url = URLEncoder.encode("https://httpbin.co/headers", "UTF-8");
Request request = new Request.Builder()
.url("https://api.scrape.do/?token=YOUR_TOKEN&url="encoded_url"&customHeaders=True")
.method("GET", body)
.addHeader("Test-Header-Key", "TestValue")
.build();
Response response = client.newCall(request).execute();string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/headers");
var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&customHeaders=true";
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
request.Headers.Add("Test-Header-Key", "TestValue");
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);If you leave the User-Agent information blank in the header field of the requests sent via programming languages, it can be filled with a default value like 'python/httpclient'. Therefore, we strongly recommend that you use a real User-Agent with this feature.
Result
{
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Authority": "httpbin.co",
"Cache-Control": "no-cache,no-cache",
"Host": "httpbin.co",
"Referer": "https://httpbin.co",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"107\", \"Chromium\";v=\"107\", \"Not=A?Brand\";v=\"24\"",
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": "\"macOS\"",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Test-Header-Key": "TestValue",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "PostmanRuntime/7.29.2",
"X-Amzn-Trace-Id": "Root=1-63748965-50104cf629bc898d03188f57"
}
}
