logo

Custom Headers

Take full control of request headers

You can interfere with all headers sent to the target website via our service. Scrape.do sends User-Agent, Accept, Cookies, etc. from you. It takes all header parameters and forwards them. You must set customHeaders=True to use the feature.

curl --location --request GET 'https://api.scrape.do/?token=YOUR_TOKEN&url=https://httpbin.co/headers&customHeaders=True'
--header 'Test-Header-Key: TestValue'
import requests
import urllib.parse

token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote("https://httpbin.co/headers")
customHeaders = "true"
payload = {}
url = "http://api.scrape.do/?token={}&url={}&customHeaders={}".format(token, targetUrl, customHeaders)
headers = {
  'Test-Header-Key': 'TestValue'
}
response = requests.request("GET", url, headers=headers, data=payload)
print(response.text)
const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/headers");
const config = {
    'method': 'GET',
    'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&customHeaders=true`,
    'headers': {
        'Test-Header-Key': 'TestValue',
    }
};
axios(config)
    .then(function (response) {
        console.log(response.data);
    })
    .catch(function (error) {
        console.log(error);
    });
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	token := "YOUR_TOKEN"
	encoded_url := url.QueryEscape("https://httpbin.co/headers")
	url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&customHeaders=true", token, encoded_url)
	method := "GET"
	client := &http.Client{}
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		fmt.Println(err)
		return
	}
	req.Header.Add("Test-Header-Key", "TestValue")
	res, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))
}
require "uri"
require "net/http"
require 'cgi'

str = CGI.escape  "https://httpbin.co/headers"
url = URI("https://api.scrape.do/?url=" + str  + "&token=YOUR_TOKEN&customHeaders=true")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
request['Test-Header-Key'] = 'TestValue'
response = https.request(request)
puts response.read_body
OkHttpClient client = new OkHttpClient().newBuilder()
  .build();
MediaType mediaType = MediaType.parse("text/plain");
RequestBody body = RequestBody.create(mediaType, "");
String encoded_url = URLEncoder.encode("https://httpbin.co/headers", "UTF-8");
Request request = new Request.Builder()
  .url("https://api.scrape.do/?token=YOUR_TOKEN&url="encoded_url"&customHeaders=True")
  .method("GET", body)
  .addHeader("Test-Header-Key", "TestValue")
  .build();
Response response = client.newCall(request).execute();
string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/headers");
var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&customHeaders=true";        
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
request.Headers.Add("Test-Header-Key", "TestValue");
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);

If you leave the User-Agent information blank in the header field of the requests sent via programming languages, it can be filled with a default value like 'python/httpclient'. Therefore, we strongly recommend that you use a real User-Agent with this feature.

Result

{
    "headers": {
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Authority": "httpbin.co",
        "Cache-Control": "no-cache,no-cache",
        "Host": "httpbin.co",
        "Referer": "https://httpbin.co",
        "Sec-Ch-Ua": "\"Google Chrome\";v=\"107\", \"Chromium\";v=\"107\", \"Not=A?Brand\";v=\"24\"",
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": "\"macOS\"",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Test-Header-Key": "TestValue",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "PostmanRuntime/7.29.2",
        "X-Amzn-Trace-Id": "Root=1-63748965-50104cf629bc898d03188f57"
    }
}