logo

Extra Headers

Add headers without replacing default headers

It is a feature that you can use only when there is one or more header parameters that you want to use for the needs of the target website, without breaking the requests we have created for the target website.

After setting the query parameter as extraHeaders=true, the header values you will send with the Sd-** prefix will be sent to the target website.

curl --location --request GET 'https://api.scrape.do/?token=YOUR_TOKEN&url=https://httpbin.co/headers&extraHeaders=true' \
--header 'sd-Test: TestValue'
import requests
import urllib.parse
token = "YOUR_TOKEN"
targetUrl = urllib.parse.quote("https://httpbin.co/headers")
extraHeaders = "true"
payload = {}
url = "http://api.scrape.do/?token={}&url={}&extraHeaders={}".format(token, targetUrl, extraHeaders)
headers = {
  'sd-Test': 'TestValue'
}
response = requests.request("GET", url, headers=headers, data=payload)
print(response.text)
const axios = require('axios');
const token = "YOUR_TOKEN";
const targetUrl = encodeURIComponent("https://httpbin.co/headers");
const extraHeaders = "true"
const config = {
    'method': 'GET',
    'url': `https://api.scrape.do/?token=${token}&url=${targetUrl}&extraHeaders=${extraHeaders}`,
    'headers': {
        'sd-Test': 'TestValue',
    }
};
axios(config)
    .then(function (response) {
        console.log(response.data);
    })
    .catch(function (error) {
        console.log(error);
    });
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	token := "YOUR_TOKEN"
	encoded_url := url.QueryEscape("https://httpbin.co/headers")
	url := fmt.Sprintf("https://api.scrape.do/?token=%s&url=%s&extraHeaders=true", token, encoded_url)
	method := "GET"
	client := &http.Client{}
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		fmt.Println(err)
		return
	}
	req.Header.Add("sd-Test", "TestValue")
	res, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))
}
require "uri"
require "net/http"
require 'cgi'

str =  CGI.escape "https://httpbin.co/headers"
url = URI("https://api.scrape.do/?url=" + str + "&token=YOUR_TOKEN&extraHeaders=true")
https = Net::HTTP.new(url.host, url.port)
https.use_ssl = true
request = Net::HTTP::Get.new(url)
request["sd-Test"] = "TestValue"
response = https.request(request)
puts response.read_body
OkHttpClient client = new OkHttpClient().newBuilder()
  .build();
MediaType mediaType = MediaType.parse("text/plain");
RequestBody body = RequestBody.create(mediaType, "");
String encoded_url = URLEncoder.encode("https://httpbin.co/headers", "UTF-8");
Request request = new Request.Builder()
  .url("https://api.scrape.do/?token=YOUR_TOKEN&url=" + encoded_url + "&extraHeaders=true")
  .method("GET", body)
  .addHeader("sd-Test", "TestValue")
  .build();
Response response = client.newCall(request).execute();
string token = "YOUR_TOKEN";
string url = WebUtility.UrlEncode("https://httpbin.co/headers");
var client = new HttpClient();
var requestURL = $"https://api.scrape.do/?token={token}&url={url}&extraHeaders=true";        
var request = new HttpRequestMessage(HttpMethod.Get, requestURL);
request.Headers.Add("sd-Test", "TestValue");
var response = client.SendAsync(request).Result;
var content = response.Content.ReadAsStringAsync().Result;
Console.WriteLine(content);

Example Result

{
    "headers": {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Authority": "httpbin.co",
        "Cache-Control": "no-cache",
        "Host": "httpbin.co",
        "Referer": "https://httpbin.co",
        "Sec-Ch-Ua": "\"Google Chrome\";v=\"107\", \"Chromium\";v=\"107\", \"Not=A?Brand\";v=\"24\"",
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": "\"macOS\"",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Test": "TestValue",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
        "X-Amzn-Trace-Id": "Root=1-63748c46-2e20cdb45d2d89455b4e4fd0"
    }
}