curl --request POST \
--url https://scrapebadger.com/v1/web/scrape \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"method": "GET",
"request_body": "<string>",
"content_type": "<string>",
"proxy_tier": "simple",
"eval_js": "<string>",
"engine": "auto",
"format": "html",
"render_js": false,
"wait_for": "<string>",
"wait_timeout": 30000,
"wait_after_load": 123,
"js_scenario": [
{}
],
"session_id": "<string>",
"retry_count": 3,
"retry_on_block": true,
"country": "<string>",
"custom_headers": {},
"screenshot": false,
"video": false,
"anti_bot": false,
"escalate": false,
"max_cost": 123,
"ai_extract": false,
"ai_prompt": "<string>"
}
'import requests
url = "https://scrapebadger.com/v1/web/scrape"
payload = {
"url": "<string>",
"method": "GET",
"request_body": "<string>",
"content_type": "<string>",
"proxy_tier": "simple",
"eval_js": "<string>",
"engine": "auto",
"format": "html",
"render_js": False,
"wait_for": "<string>",
"wait_timeout": 30000,
"wait_after_load": 123,
"js_scenario": [{}],
"session_id": "<string>",
"retry_count": 3,
"retry_on_block": True,
"country": "<string>",
"custom_headers": {},
"screenshot": False,
"video": False,
"anti_bot": False,
"escalate": False,
"max_cost": 123,
"ai_extract": False,
"ai_prompt": "<string>"
}
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
method: 'GET',
request_body: '<string>',
content_type: '<string>',
proxy_tier: 'simple',
eval_js: '<string>',
engine: 'auto',
format: 'html',
render_js: false,
wait_for: '<string>',
wait_timeout: 30000,
wait_after_load: 123,
js_scenario: [{}],
session_id: '<string>',
retry_count: 3,
retry_on_block: true,
country: '<string>',
custom_headers: {},
screenshot: false,
video: false,
anti_bot: false,
escalate: false,
max_cost: 123,
ai_extract: false,
ai_prompt: '<string>'
})
};
fetch('https://scrapebadger.com/v1/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://scrapebadger.com/v1/web/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'method' => 'GET',
'request_body' => '<string>',
'content_type' => '<string>',
'proxy_tier' => 'simple',
'eval_js' => '<string>',
'engine' => 'auto',
'format' => 'html',
'render_js' => false,
'wait_for' => '<string>',
'wait_timeout' => 30000,
'wait_after_load' => 123,
'js_scenario' => [
[
]
],
'session_id' => '<string>',
'retry_count' => 3,
'retry_on_block' => true,
'country' => '<string>',
'custom_headers' => [
],
'screenshot' => false,
'video' => false,
'anti_bot' => false,
'escalate' => false,
'max_cost' => 123,
'ai_extract' => false,
'ai_prompt' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://scrapebadger.com/v1/web/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://scrapebadger.com/v1/web/scrape")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://scrapebadger.com/v1/web/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"url": "https://scrapebadger.com",
"status_code": 200,
"content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
"format": "markdown",
"engine_used": "http",
"credits_used": 2,
"duration_ms": 342,
"retries_used": 0,
"content_length": 1256,
"screenshot_url": null,
"video_url": null,
"headers": {
"content-type": "text/html; charset=UTF-8"
},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": null,
"ai_model": null,
"ai_error": null
}
{
"success": true,
"url": "https://scrapebadger.com/products",
"status_code": 200,
"content": "# Products\n\n...",
"format": "markdown",
"engine_used": "http",
"credits_used": 4,
"duration_ms": 2150,
"retries_used": 0,
"content_length": 8432,
"screenshot_url": null,
"video_url": null,
"headers": {},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": [
{ "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
{ "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
],
"ai_model": "gpt-4o-mini",
"ai_error": null
}
{
"error": "blocking_page_detected",
"detail": "All retry attempts exhausted — blocking page still detected.",
"block_kind": "captcha",
"vendor": "perimeterx",
"vendors": ["perimeterx"],
"unblock": "discovering",
"captcha": {
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
},
"data": {
"success": false,
"url": "https://protected-site.com/",
"status_code": 403,
"engine_used": "http",
"credits_used": 0,
"retries_used": 3,
"blocking_detected": true,
"blocking_details": {
"is_blocked": true,
"block_type": "perimeterx",
"confidence": 0.95,
"details": "PerimeterX block interstitial detected",
"vendor": "perimeterx",
"kind": "captcha",
"signals": ["px-captcha", "press & hold", "access denied page"]
},
"antibot_systems": [
{ "system": "perimeterx", "confidence": 0.95, "details": "_px cookies; px-captcha" }
],
"captcha_systems": [
{
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
}
]
}
}
Scrape URL
Scrape a webpage and return its content as HTML, Markdown, or plain text.
curl --request POST \
--url https://scrapebadger.com/v1/web/scrape \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"method": "GET",
"request_body": "<string>",
"content_type": "<string>",
"proxy_tier": "simple",
"eval_js": "<string>",
"engine": "auto",
"format": "html",
"render_js": false,
"wait_for": "<string>",
"wait_timeout": 30000,
"wait_after_load": 123,
"js_scenario": [
{}
],
"session_id": "<string>",
"retry_count": 3,
"retry_on_block": true,
"country": "<string>",
"custom_headers": {},
"screenshot": false,
"video": false,
"anti_bot": false,
"escalate": false,
"max_cost": 123,
"ai_extract": false,
"ai_prompt": "<string>"
}
'import requests
url = "https://scrapebadger.com/v1/web/scrape"
payload = {
"url": "<string>",
"method": "GET",
"request_body": "<string>",
"content_type": "<string>",
"proxy_tier": "simple",
"eval_js": "<string>",
"engine": "auto",
"format": "html",
"render_js": False,
"wait_for": "<string>",
"wait_timeout": 30000,
"wait_after_load": 123,
"js_scenario": [{}],
"session_id": "<string>",
"retry_count": 3,
"retry_on_block": True,
"country": "<string>",
"custom_headers": {},
"screenshot": False,
"video": False,
"anti_bot": False,
"escalate": False,
"max_cost": 123,
"ai_extract": False,
"ai_prompt": "<string>"
}
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
method: 'GET',
request_body: '<string>',
content_type: '<string>',
proxy_tier: 'simple',
eval_js: '<string>',
engine: 'auto',
format: 'html',
render_js: false,
wait_for: '<string>',
wait_timeout: 30000,
wait_after_load: 123,
js_scenario: [{}],
session_id: '<string>',
retry_count: 3,
retry_on_block: true,
country: '<string>',
custom_headers: {},
screenshot: false,
video: false,
anti_bot: false,
escalate: false,
max_cost: 123,
ai_extract: false,
ai_prompt: '<string>'
})
};
fetch('https://scrapebadger.com/v1/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://scrapebadger.com/v1/web/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'method' => 'GET',
'request_body' => '<string>',
'content_type' => '<string>',
'proxy_tier' => 'simple',
'eval_js' => '<string>',
'engine' => 'auto',
'format' => 'html',
'render_js' => false,
'wait_for' => '<string>',
'wait_timeout' => 30000,
'wait_after_load' => 123,
'js_scenario' => [
[
]
],
'session_id' => '<string>',
'retry_count' => 3,
'retry_on_block' => true,
'country' => '<string>',
'custom_headers' => [
],
'screenshot' => false,
'video' => false,
'anti_bot' => false,
'escalate' => false,
'max_cost' => 123,
'ai_extract' => false,
'ai_prompt' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://scrapebadger.com/v1/web/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://scrapebadger.com/v1/web/scrape")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://scrapebadger.com/v1/web/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"method\": \"GET\",\n \"request_body\": \"<string>\",\n \"content_type\": \"<string>\",\n \"proxy_tier\": \"simple\",\n \"eval_js\": \"<string>\",\n \"engine\": \"auto\",\n \"format\": \"html\",\n \"render_js\": false,\n \"wait_for\": \"<string>\",\n \"wait_timeout\": 30000,\n \"wait_after_load\": 123,\n \"js_scenario\": [\n {}\n ],\n \"session_id\": \"<string>\",\n \"retry_count\": 3,\n \"retry_on_block\": true,\n \"country\": \"<string>\",\n \"custom_headers\": {},\n \"screenshot\": false,\n \"video\": false,\n \"anti_bot\": false,\n \"escalate\": false,\n \"max_cost\": 123,\n \"ai_extract\": false,\n \"ai_prompt\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"url": "https://scrapebadger.com",
"status_code": 200,
"content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
"format": "markdown",
"engine_used": "http",
"credits_used": 2,
"duration_ms": 342,
"retries_used": 0,
"content_length": 1256,
"screenshot_url": null,
"video_url": null,
"headers": {
"content-type": "text/html; charset=UTF-8"
},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": null,
"ai_model": null,
"ai_error": null
}
{
"success": true,
"url": "https://scrapebadger.com/products",
"status_code": 200,
"content": "# Products\n\n...",
"format": "markdown",
"engine_used": "http",
"credits_used": 4,
"duration_ms": 2150,
"retries_used": 0,
"content_length": 8432,
"screenshot_url": null,
"video_url": null,
"headers": {},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": [
{ "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
{ "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
],
"ai_model": "gpt-4o-mini",
"ai_error": null
}
{
"error": "blocking_page_detected",
"detail": "All retry attempts exhausted — blocking page still detected.",
"block_kind": "captcha",
"vendor": "perimeterx",
"vendors": ["perimeterx"],
"unblock": "discovering",
"captcha": {
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
},
"data": {
"success": false,
"url": "https://protected-site.com/",
"status_code": 403,
"engine_used": "http",
"credits_used": 0,
"retries_used": 3,
"blocking_detected": true,
"blocking_details": {
"is_blocked": true,
"block_type": "perimeterx",
"confidence": 0.95,
"details": "PerimeterX block interstitial detected",
"vendor": "perimeterx",
"kind": "captcha",
"signals": ["px-captcha", "press & hold", "access denied page"]
},
"antibot_systems": [
{ "system": "perimeterx", "confidence": 0.95, "details": "_px cookies; px-captcha" }
],
"captcha_systems": [
{
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
}
]
}
}
Request Body
GET, POST, PUT, PATCH, DELETE, HEAD
or OPTIONS. POST, PUT and PATCH require request_body.POST/PUT/PATCH. Pass it as a string, and
set content_type to match — there is no form_data parameter.{
"url": "https://example.com/login",
"method": "POST",
"request_body": "username=alice&password=hunter2",
"content_type": "application/x-www-form-urlencoded"
}
{
"url": "https://example.com/api/search",
"method": "POST",
"request_body": "{\"q\":\"shoes\"}",
"content_type": "application/json"
}
| Value | Engine cost | Description |
|---|---|---|
auto | from 1 credit | Automatically picks the best engine for the target site (recommended) |
browser | 5 credits | Force headless browser with full JavaScript rendering |
auto mode, simple pages use fast HTTP (1 credit) and JavaScript-heavy
pages use a browser (5 credits).proxy_tier surcharge, so the cheapest possible scrape
is 2 credits, not 1, and a browser scrape is 6, not 5. See
Credit costs for the full arithmetic.| Value | Pool | Surcharge |
|---|---|---|
simple | Standard shared pool — the cheapest, with the fewest guarantees about IP type and reputation | +1 credit |
premium | Residential IPs | +3 credits |
ultra | Residential IPs from our highest-trust providers | +8 credits |
simple unless the target actually blocks requests from it — moving
to premium makes a browser scrape 8 credits instead of 6. Use
max_cost if you want a hard ceiling. Country coverage
differs slightly per tier: see Proxy countries.html— Raw HTML of the pagemarkdown— Converted to clean Markdowntext— Plain text with HTML tags stripped
raw format — to get an
unwrapped body, use raw_content.browser engine. Use this for single-page applications or pages that load content dynamically.browser engine. If render_js is false and this is set, JS rendering is forced automatically.The wait runs after js_scenario, so a login flow can fill the form,
submit, and then wait for an element that only exists on the post-login page.
The wait is a precondition, not a hint: if the selector has not appeared
within wait_timeout, the request fails with 422 wait_for_timeout, nothing
is charged, and the response carries js_scenario_report so you can see how
far the flow got. A successful response reports wait_for_found: true. For a
best-effort dwell that never fails, use wait_after_load instead.{ "wait_for": "#main-content" }
{ "wait_for": "//div[@class='results']" }
render_js alone is often not enough. It returns the DOM once the page
has loaded — which for anything drawn by a third-party widget (an embedded
login form, a consent gate, a chat panel, a payment iframe) is the empty
container, not the content. Those widgets fetch and paint seconds after load.If a form or list is missing from your result, name it in wait_for rather
than raising wait_after_load; the selector returns as soon as the element
exists instead of always paying a fixed dwell.{
"url": "https://example.com/login",
"render_js": true,
"wait_for": "#login-container input[type=password]",
"wait_timeout": 30000
}
wait_for selector to appear. Range: 1000 – 120000.0 – 30000.wait_for is awaited and content is extracted. Forces the browser
engine. Each step is an object with an action and action-specific
parameters.Supported actions:| Action | Parameters | Description |
|---|---|---|
click | selector | Click an element |
fill | selector, value | Click into an input and type value with real key events |
scroll | amount (optional, px) | Scroll down; a random 300–700 px when amount is omitted |
wait | timeout (ms) | Pause for a fixed duration |
evaluate | script | Run JavaScript in the page |
{
"js_scenario": [
{ "action": "fill", "selector": "#username", "value": "me@example.com" },
{ "action": "fill", "selector": "#password", "value": "hunter2" },
{ "action": "click", "selector": "button[type=submit]" },
{ "action": "wait", "timeout": 2000 }
]
}
422 js_scenario_failed and nothing is charged. Every response
that ran a scenario includes js_scenario_report, one entry per executed step
with its outcome and the page URL after it — the log to read when a flow does
not end where you expected. An unknown action is rejected up front with a
validation error rather than skipped. (type and milliseconds are accepted
as aliases of action and timeout.)session_id to maintain login state or continue a browsing session.{ "session_id": "my-shopping-session" }
0 – 10.false to get the blocked response immediately.{ "country": "US" }
KP), the scrape fails instead
of falling back to another country: HTTP 200 with success: false and
error: "fetch_failed", charged 0 credits. A
per-host override that pins a
country takes precedence over yours. See
Proxy countries for which countries had exits
on each proxy_tier when we last measured.{
"custom_headers": {
"Accept-Language": "en-US",
"Referer": "https://google.com"
}
}
content (objects are
JSON-serialised). May be an async arrow function. Browser engines only —
forces the browser engine. No extra credits.Use it to pull something out of the live page that the HTML does not carry:{ "url": "https://example.com", "render_js": true, "eval_js": "document.cookie" }
{
"url": "https://example.com",
"render_js": true,
"eval_js": "(async () => JSON.stringify({ id: window.__APP_STATE__.id }))()"
}
document.cookie returns only non-HttpOnly cookies — that is a browser
rule, not a ScrapeBadger limit. Anti-bot vendors’ session cookies are also
bound to the IP and TLS fingerprint that minted them, so replaying them from
a different machine does not carry the session. To keep a session alive
across several scrapes, use session_id instead and
keep the whole flow on our side.screenshot_url response field as a data:image/png;base64,… URI. Captures the viewport unless screenshot_full_page is set. For a screenshot-only call, POST /v1/web/screenshot returns the bare base64 PNG.screenshot: true, capture the whole scrollable page instead of the viewport.video_url response field as a data:image/gif;base64,… URI. No extra charge — you pay the browser engine cost you were already paying. Useful for debugging, visual verification, or monitoring how a page loads.The recording is bounded so it stays small enough to return inline: up to 5 frames per second, at most 640 px on its longest side, idle stretches (a wait_for or a dwell) shortened to 3 s, and at most 2 MB. A recording over 2 MB is downscaled, then thinned to fewer frames. Recording never fails the scrape: if nothing could be recorded you still get the page, with video_url: null.http → browserYou only pay for the engine that succeeds — costs are not cumulative. Without this flag, only the selected engine is tried.400 error if the estimated cost would exceed this budget. Useful for controlling costs when using escalate or anti_bot. Minimum: 1.X-Scrape-* response headers.Use it for two things: skipping the JSON encode/decode on large HTML payloads
(saves 300–1000 ms on 1 MB+ responses), and downloading binary files
without the ~33% base64 overhead. See
Binary files and raw bodies.Cannot be combined with ai_extract, screenshot or video — those need the
JSON envelope, so the request falls back to it automatically.ai_prompt. Adds +2 credits to the request cost. The scrape result is still returned even if AI extraction fails, and a failed extraction is not charged.ai_extract is true. Maximum 2000 characters.{
"ai_extract": true,
"ai_prompt": "Extract all product names and prices as a JSON array"
}
Credit costs
Every request is billed as engine + proxy tier + options. The proxy tier is charged on every request, which is why nothing costs 1 credit in practice.| Component | Credits |
|---|---|
HTTP engine (http) | 1 |
Browser engine (browser) | 5 |
proxy_tier: simple (default) | +1 |
proxy_tier: premium | +3 |
proxy_tier: ultra | +8 |
ai_extract | +2 |
anti_bot, when a solver is actually invoked | +5 |
screenshot, video, js_scenario, eval_js, wait_for | +0 |
| Request | Total |
|---|---|
| Simple page, default tier | 2 |
Simple page, premium tier | 4 |
Simple page, ultra tier | 9 |
| JS-rendered page, default tier | 6 |
JS-rendered page, premium tier | 8 |
Simple page + ai_extract | 4 |
X-Credits-Used response header and
the credits_used body field — trust those over any estimate. A request
rejected before scraping (400, 422) is charged 0.
max_cost to put a ceiling on a request. It is checked
against the engine’s cost, plus anything a per-host override adds (a higher
proxy tier than you asked for, or a browser engine). The proxy tier you chose
yourself is not part of the check, so compare max_cost with the engine price.Per-host overrides and automatic unblocking
Some sites only answer a specific setup: a particular proxy tier or country, a warm-up visit, or a real browser. ScrapeBadger keeps a working configuration per host and applies it for you:- On a host with an override, its access settings (engine chain, proxy
tier/type/country, warm-up and injected headers) are used instead of yours.
Your output settings (
render_js,wait_for,js_scenario,format,screenshot, …) are still honoured. You are billed for the configuration actually used; theX-Proxy-Tier-Effectiveresponse header names the tier. - On a host without one, your parameters are used as sent. If the host keeps
blocking requests, ScrapeBadger starts searching in the background for the
cheapest configuration that reliably gets through, and applies it to later
requests automatically. The blocked request still returns its
422straight away, with the search state in theX-Unblockheader and theunblockbody field:discovering(a search is running),cooldown(a recent search found nothing; retried later) ornone. - When an override stops working, the search runs again and replaces it only if the new configuration does measurably better.
Response
false when all retries are exhausted and the page is still blocked.null when success is false,
and null for binary targets — those come back in content_base64.content when the target
serves a binary payload. Only present when is_binary is true. Bodies above
25 MB are not base64-encoded — re-request those with raw_content: true.proxy_tier surcharge,
plus solver and AI extraction when used. See Credit costs.
This is the authoritative figure and matches the X-Credits-Used header.data:image/png;base64,… URI. Only present when screenshot: true was requested.data:image/gif;base64,… URI. Only present when video: true was requested; null if the session could not be recorded.blocking_detected is true.Show child attributes
Show child attributes
cloudflare, datadome, akamai, kasada, forbidden_shell: an HTTP 403 whose page is only scripts, with no content, rate_limited: an HTTP 429 that refuses the request instead of returning the page, or login_wall: the site redirected to its login page instead of the requested page). Blocked responses, these included, are not charged.cloudflare, datadome, akamai, kasada, incapsula, perimeterx, aws_waf, f5, peakhour, reblaze, castle, radware, ddos_guard, sucuri, anubis, vercel, netacea, alibaba, servicepipe, google, fastly or other. Omitted when unknown.antibot, captcha, rate_limit, ip_ban, geo_block, auth_wall, empty (an empty or script-only page) or other.Show child attributes
Show child attributes
recaptcha_v2, recaptcha_v3, hcaptcha, geetest, cloudflare_turnstile, funcaptcha, aws_waf_captcha, friendly_captcha, mtcaptcha, yandex_smartcaptcha, tencent_captcha, alibaba_slider, lemin, datadome_captcha, perimeterx_press_hold or image_captcha.v2_checkbox, v2_invisible, v3 or enterprise (reCAPTCHA), v3 or v4 (GeeTest), standalone or challenge (Turnstile).true when this CAPTCHA is what blocks the page. A widget embedded in an ordinary page, such as a login or contact form, is reported with gate: false and does not count as a block.wait_for selector appeared within wait_timeout. null when no
wait_for was requested. A miss never reaches you as a 200 — it is a
422 wait_for_timeout.js_scenario step, in order. null when no scenario
ran. Execution stops at the first failed step, so a report shorter than the
scenario means a step broke.Show child attributes
Show child attributes
ai_prompt. The shape depends on your prompt. null when ai_extract is false or extraction failed.Binary files and raw bodies
/v1/web/scrape handles binary targets — images, PDFs, archives, fonts, audio
and video — as well as HTML. The bytes are returned exactly as the origin sent
them; nothing is decoded, parsed or re-encoded on the way through.
This works through the same anti-bot machinery as a page scrape, so a file
behind Cloudflare, DataDome or Imperva is fetched with the same engine, proxy
tier and session you would use for its parent page. Pass the same session_id
you used to scrape the page the file was linked from and the file download
reuses that session’s cookies and fingerprint.
There are two ways to get the bytes.
raw_content: true — the body itself. No base64 overhead. Best for large
files and for piping straight to disk.
curl -X POST "https://scrapebadger.com/v1/web/scrape" \
-H "X-API-Key: YOUR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"url": "https://example.com/photos/item.jpg",
"raw_content": true,
"session_id": "gallery-session"
}' \
--output item.jpg
import requests
response = requests.post(
"https://scrapebadger.com/v1/web/scrape",
headers={"X-API-Key": "YOUR_API_KEY"},
json={
"url": "https://example.com/photos/item.jpg",
"raw_content": True,
"session_id": "gallery-session",
},
)
with open("item.jpg", "wb") as f:
f.write(response.content) # exact bytes — do NOT use response.text
print(response.headers["Content-Type"])
print(response.headers["X-Credits-Used"])
const response = await fetch("https://scrapebadger.com/v1/web/scrape", {
method: "POST",
headers: {
"X-API-Key": "YOUR_API_KEY",
"Content-Type": "application/json",
},
body: JSON.stringify({
url: "https://example.com/photos/item.jpg",
raw_content: true,
session_id: "gallery-session",
}),
});
const buffer = Buffer.from(await response.arrayBuffer());
response.text in Python or
response.text() in JavaScript will decode the payload as a string and
corrupt it. Use response.content / response.arrayBuffer().content_base64 — one request, metadata included. Use this when
you also want credits_used, engine_used or the protection detections
alongside the file.
import base64
import requests
data = requests.post(
"https://scrapebadger.com/v1/web/scrape",
headers={"X-API-Key": "YOUR_API_KEY"},
json={"url": "https://example.com/report.pdf"},
).json()
if data["is_binary"]:
with open("report.pdf", "wb") as f:
f.write(base64.b64decode(data["content_base64"]))
print(data["content_type"]) # application/pdf
else:
print(data["content"]) # a text target — HTML/markdown/text
raw_content always carry
Content-Disposition: attachment and X-Content-Type-Options: nosniff, and
their Content-Type is restricted to a known-safe set — scraped bytes are
never labelled in a way that would let a browser execute them.
Bodies larger than 25 MB are not base64-encoded into JSON. Those return
content_base64: null with a detail telling you to re-request using
raw_content: true, which returns them without the base64 expansion. Very
large files are still held in memory end to end, so treat a few hundred MB as
the practical ceiling for a single request.
Examples
Basic scrape
curl -X POST "https://scrapebadger.com/v1/web/scrape" \
-H "x-api-key: YOUR_API_KEY" \
-H "Content-Type: application/json" \
-d '{"url": "https://scrapebadger.com", "format": "markdown"}'
import requests
response = requests.post(
"https://scrapebadger.com/v1/web/scrape",
headers={"x-api-key": "YOUR_API_KEY"},
json={"url": "https://scrapebadger.com", "format": "markdown"}
)
print(response.json()["content"])
const res = await fetch("https://scrapebadger.com/v1/web/scrape", {
method: "POST",
headers: { "x-api-key": "YOUR_API_KEY", "Content-Type": "application/json" },
body: JSON.stringify({ url: "https://scrapebadger.com", format: "markdown" })
});
const data = await res.json();
console.log(data.content);
JavaScript rendering with wait
{
"url": "https://scrapebadger.com/spa-page",
"format": "html",
"render_js": true,
"wait_for": "#dynamic-content",
"wait_timeout": 10000
}
AI extraction
{
"url": "https://scrapebadger.com/products",
"format": "markdown",
"ai_extract": true,
"ai_prompt": "Extract all product names, prices, and ratings as a JSON array of objects with keys: name, price, rating"
}
Full anti-bot bypass with budget
{
"url": "https://heavily-protected-site.com",
"format": "markdown",
"escalate": true,
"anti_bot": true,
"max_cost": 20,
"country": "US"
}
Browser automation scenario
Log in, then wait for an element that only exists once the login succeeded. If the flow never gets there you receive a free422 with the step-by-step
report instead of a billed copy of the login page.
{
"url": "https://example.com/login",
"js_scenario": [
{ "action": "fill", "selector": "#username", "value": "me@example.com" },
{ "action": "fill", "selector": "#password", "value": "hunter2" },
{ "action": "click", "selector": "button[type=submit]" }
],
"wait_for": "#account-dashboard",
"wait_timeout": 60000
}
{
"url": "https://scrapebadger.com/infinite-scroll",
"format": "text",
"js_scenario": [
{ "action": "scroll", "amount": 1000 },
{ "action": "wait", "timeout": 2000 },
{ "action": "scroll", "amount": 1000 },
{ "action": "wait", "timeout": 2000 }
]
}
Error Responses
| Status | Description |
|---|---|
200 with success: false | error: "fetch_failed": no response from the target at all, including when no proxy exit is available in the requested country. Never charged. |
400 | Invalid URL (including a hostname confirmed not to exist), cost exceeds max_cost, or requested engine not available. Never charged. |
402 | Insufficient credits |
422 | Blocking detected after all retries exhausted (error: "blocking_page_detected", with block_kind, vendor, vendors, unblock and, for a CAPTCHA wall, the gating captcha at the top level; the scrape result is in data). vendors lists every protection vendor seen on the page, the block’s own first: a site can put two in front of one page, such as a Cloudflare challenge before PerimeterX. A request that runs out of time while retrying (about 165 s) returns this 422 rather than a gateway timeout; the wait_for selector never appeared (error: "wait_for_timeout"); a js_scenario step failed (error: "js_scenario_failed"); or the body contained an unknown field (error: "unknown_request_field"). Never charged. |
429 | Rate limit exceeded |
500 | Unexpected server error |
Hostname does not exist
If two independent DNS resolvers confirm that your URL’s hostname does not exist (NXDOMAIN), the request returns 400 with error: "Invalid URL"
before scraping. You are charged 0 credits (X-Credits-Used: 0). Check
the hostname for typos and confirm that the domain exists before retrying.
A hostname with no address records (NODATA) is not enough to trigger this
rejection. DNS timeouts, SERVFAIL, other resolver errors, and disagreement
between resolvers also do not trigger it; the request continues through the
normal scraping and retry path.
Unknown request fields
A field this endpoint does not define is rejected rather than ignored, so a request never looks like it succeeded at something it did not do. The response names the offending field and lists everything the endpoint accepts, and nothing is charged.{
"error": "unknown_request_field",
"detail": "Unknown request field(s): return_cookies. These were silently ignored by older versions; they are now rejected so a request never appears to succeed at something it did not do.",
"unknown_fields": ["return_cookies"],
"accepted_fields": ["ai_extract", "ai_prompt", "anti_bot", "..."]
}
return_cookies or form_data parameter. To read cookies use
eval_js; to send a request body use
method with request_body.
{
"success": true,
"url": "https://scrapebadger.com",
"status_code": 200,
"content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
"format": "markdown",
"engine_used": "http",
"credits_used": 2,
"duration_ms": 342,
"retries_used": 0,
"content_length": 1256,
"screenshot_url": null,
"video_url": null,
"headers": {
"content-type": "text/html; charset=UTF-8"
},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": null,
"ai_model": null,
"ai_error": null
}
{
"success": true,
"url": "https://scrapebadger.com/products",
"status_code": 200,
"content": "# Products\n\n...",
"format": "markdown",
"engine_used": "http",
"credits_used": 4,
"duration_ms": 2150,
"retries_used": 0,
"content_length": 8432,
"screenshot_url": null,
"video_url": null,
"headers": {},
"blocking_detected": false,
"blocking_details": null,
"antibot_systems": [],
"captcha_systems": [],
"anti_bot_solved": false,
"solver_used": null,
"ai_extraction": [
{ "name": "Widget Pro", "price": "$29.99", "rating": 4.5 },
{ "name": "Widget Basic", "price": "$9.99", "rating": 4.2 }
],
"ai_model": "gpt-4o-mini",
"ai_error": null
}
{
"error": "blocking_page_detected",
"detail": "All retry attempts exhausted — blocking page still detected.",
"block_kind": "captcha",
"vendor": "perimeterx",
"vendors": ["perimeterx"],
"unblock": "discovering",
"captcha": {
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
},
"data": {
"success": false,
"url": "https://protected-site.com/",
"status_code": 403,
"engine_used": "http",
"credits_used": 0,
"retries_used": 3,
"blocking_detected": true,
"blocking_details": {
"is_blocked": true,
"block_type": "perimeterx",
"confidence": 0.95,
"details": "PerimeterX block interstitial detected",
"vendor": "perimeterx",
"kind": "captcha",
"signals": ["px-captcha", "press & hold", "access denied page"]
},
"antibot_systems": [
{ "system": "perimeterx", "confidence": 0.95, "details": "_px cookies; px-captcha" }
],
"captcha_systems": [
{
"system": "perimeterx_press_hold",
"confidence": 0.95,
"details": "px-captcha element; 'Press & Hold' prompt; _pxAppId",
"gate": true,
"params": { "app_id": "PXxOR1K5b6", "page_url": "https://protected-site.com/" }
}
]
}
}
Body
Proxy pool to route through: simple = standard shared pool (cheapest, fewest guarantees about IP type), premium = residential IPs, ultra = residential IPs from our highest-trust providers. Its surcharge is added to EVERY request: simple +1, premium +3, ultra +8 credits.
simple, premium, ultra JavaScript evaluated in the loaded page after navigation and waits; its return value replaces the page HTML in content. Browser engines only. Use it to read what the HTML does not carry, e.g. document.cookie (non-HttpOnly cookies only, and anti-bot cookies are bound to the IP and TLS fingerprint that minted them).
ISO 3166-1 alpha-2 country code for the proxy exit; the request goes out from an IP in that country (default US). Any alpha-2 code is accepted, with no allow-list. Targeting is country-level, not city-level. If no exit is available in that country the scrape fails (HTTP 200, success=false, error=fetch_failed) and is not billed; it never falls back to another country. A per-host override that pins a country takes precedence. Measured coverage per proxy tier: https://docs.scrapebadger.com/web-scraping/proxy-countries
Response
Successful scrape

