Send your first request

Send your first request using Thordata’s Web Scraper API. Before you start, you need an API Token. You can obtain a free trial token on the [API Builder] page. Then, copy your credentials from the “Token” field. Note: The token you receive is sensitive information. Please keep it secure and do not disclose it. If your token is leaked and your resources are abused, you will bear the consequences yourself; the platform is not responsible. Code Example: After obtaining your API credentials, use the following code to send your first request:

cURL-Linux

curl -X POST "http://scraperapi.thordata.com/builder" \
  -H "Authorization: Bearer Token-ID" \
  -H "Content-Type: application/x-www-form-urlencoded" \
  -d "spider_name=amazon.com" \
  -d "individual_params={\"spider_parameters\":[{\"url\": \"https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726\",\"zip_code\": \"94107\"}],\"spider_id\":\"amazon_product_by-url\",\"spider_errors\":true}" \
  -d "file_name={{TasksID}}"

cURL-Windows

curl -X POST "http://scraperapi.thordata.com/builder" ^
  -H "Authorization: Bearer Token-ID" ^
  -H "Content-Type: application/x-www-form-urlencoded" ^
  -d "spider_name=amazon.com" ^
  -d "individual_params={\"spider_parameters\":[{\"url\": \"https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726\",\"zip_code\": \"94107\"}],\"spider_id\":\"amazon_product_by-url\",\"spider_errors\":true}" ^
  -d "file_name={{TasksID}}"

Python

import requests
import json

def main():
  client = requests.Session()
  target_url = "http://scraperapi.thordata.com/builder"

  individual_params = {
    "spider_parameters": [
      {
        "url": "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726",
        "zip_code": "25662"
      }
    ],
    "spider_id": "amazon_product_by-url",
    "spider_errors": True
  }

  individual_params_json = json.dumps(individual_params)
  
  form_data = {
    "spider_name": "amazon.com",
    "individual_params": individual_params_json,
    "file_name": "{{TasksID}}"
  }

  headers = {
    "Authorization": "Bearer Token-ID",
    "Content-Type": "application/x-www-form-urlencoded"
  }

  try:
    resp = client.post(target_url, data=form_data, headers=headers)
    resp.raise_for_status()  # Raises an HTTPError for bad responses
    
    print(f"Status Code: {resp.status_code}")
    print(f"Response Body: {resp.text}")
      
  except requests.exceptions.RequestException as e:
    print(f"Error sending request: {e}")

if __name__ == "__main__":
  main()

C#

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Web;

class Program
{
  static async Task Main(string[] args)
  {
    var client = new HttpClient();
    var targetUrl = "http://scraperapi.thordata.com/builder";

    var spiderInfo =  new
    {
      spider_parameters = new List<Dictionary<string, string>>
      {
        new Dictionary<string, string>
        {
          {"url", "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726"},
          {"zip_code", "25662"}
        }
      },
      spider_id = "amazon_product_by-url",
      spider_errors = true
    };
    
    var spiderInfoJson = Newtonsoft.Json.JsonConvert.SerializeObject(spiderInfo);
    
    var formData = new Dictionary<string, string>
    {
      { "spider_name", "amazon.com" },
      { "individual_params", spiderInfoJson },
      { "file_name", "{{TasksID}}"}
    };

    var request = new HttpRequestMessage(HttpMethod.Post, targetUrl)
    {
      Content = new FormUrlEncodedContent(formData)
    };

    request.Headers.Add("Authorization", "Bearer Token-ID");

    try
    {
      var response = await client.SendAsync(request);
      var responseBody = await response.Content.ReadAsStringAsync();

      Console.WriteLine($"Status Code: {(int)response.StatusCode}");
      Console.WriteLine($"Response Body: {responseBody}");
    }
    catch (HttpRequestException e)
    {
      Console.WriteLine($"Error sending request: {e.Message}");
    }
  }
}

GO

package main

import (
  "encoding/json"
  "fmt"
  "io"
  "net/http"
  "net/url"
  "strings"
)

func main() {
  client := &http.Client{}
  targetURL := "http://scraperapi.thordata.com/builder"

  spiderInfo := map[string]interface{}{
    "spider_parameters": []map[string]string{ 
      {
        "url": "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726",
        "zip_code": "25662",
      },
    },
    "spider_id": "amazon_product_by-url",
    "spider_errors": true,
  }

  spiderInfoJSON, err := json.Marshal(spiderInfo)
  if err != nil {
    fmt.Println("Error marshaling JSON:", err)
    return
  }
  
  formData := url.Values{}
  formData.Add("spider_name", "amazon.com")
  formData.Add("individual_params", string(spiderInfoJSON))
  formData.Add("file_name", "{{TasksID}}")
  
  req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode()))
  if err != nil {
    fmt.Println("Error creating request:", err)
    return
  }

  req.Header.Set("Authorization", "Bearer Token-ID")
  req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

  resp, err := client.Do(req)
  if err != nil {
    fmt.Println("Error sending request:", err)
    return
  }
  defer resp.Body.Close()

  body, err := io.ReadAll(resp.Body)
  if err != nil {
    fmt.Println("Error reading response:", err)
    return
  }

  fmt.Printf("Status Code: %d\n", resp.StatusCode)
  fmt.Printf("Response Body: %s\n", body)
}

Node

const axios = require('axios');
const qs = require('querystring');

async function main() {
  const targetURL = "http://scraperapi.thordata.com/builder";

  const spiderInfo = {
    spider_parameters: [ 
      {
        url: "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726",
        zip_code: "25662"
      }
    ],
    spider_id: "amazon_product_by-url",
    spider_errors: true
  };
  
  try {
    const spiderInfoJSON = JSON.stringify(spiderInfo);
    
    const formData = qs.stringify({
      spider_name: "amazon.com",
      individual_params: spiderInfoJSON,
      file_name: "{{TasksID}}"
    });

    const response = await axios.post(targetURL, formData, {
      headers: {
        'Authorization': 'Bearer Token-ID',
        'Content-Type': 'application/x-www-form-urlencoded'
      }
    });

    console.log(`Status Code: ${ response.status } `);
    console.log(`Response Body: ${ JSON.stringify(response.data) } `);
  } catch (error) {
    if (error.response) {
      console.error(`Error response: ${ error.response.status } - ${ error.response.data }`);
    } else if (error.request) {
      console.error('No response received:', error.request);
    } else {
      console.error('Error:', error.message);
    }
  }
}

main();

PHP

<?php

$targetURL = "http://scraperapi.thordata.com/builder";

$spiderInfo = [
  "spider_parameters" => [
    [
      "url" => "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726",
      "zip_code" => "25662"
    ]
  ],
  "spider_id" => "amazon_product_by-url",
  "spider_errors" => true
];

$spiderInfoJSON = json_encode($spiderInfo);
if ($spiderInfoJSON === false) {
  die("Error marshaling JSON: " . json_last_error_msg());
}

$formData = [
  "spider_name" => "amazon.com",
  "individual_params" => $spiderInfoJSON,
  "file_name" => "{{TasksID}}"
];

$options = [
  "http" => [
    "method" => "POST",
    "header" => implode("\r\n", [
      "Authorization: Bearer Token-ID",
      "Content-Type: application/x-www-form-urlencoded",
      "Accept: application/json"
    ]),
    "content" => http_build_query($formData)
  ]
];

$context = stream_context_create($options);
$response = file_get_contents($targetURL, false, $context);

if ($response === false) {
  $error = error_get_last();
  die("Error sending request: " . $error["message"]);
}

// Get HTTP status code
$statusCode = null;
if (isset($http_response_header[0])) {
  preg_match('/HTTP\/\d\.\d (\d{3})/', $http_response_header[0], $matches);
  $statusCode = $matches[1] ?? null;
}

echo "Status Code: " . $statusCode . "\n";
echo "Response Body: " . $response . "\n";
?>

Java

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class Main {
  public static void main(String[] args) {
    String targetURL = "http://scraperapi.thordata.com/builder";

    // Create the spider info data structure
    Map<String, Object> spiderInfo = new HashMap<>();
    spiderInfo.put("spider_id", "amazon_product_by-url");
    spiderInfo.put("spider_errors", true);

    List<Map<String, String>> spiderParameters = new ArrayList<>();
    
    Map<String, String> param1 = new HashMap<>();
    param1.put("url", "https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726");
    param1.put("zip_code", "25662");
    spiderParameters.add(param1);

    spiderInfo.put("spider_parameters", spiderParameters);
    
    
    try {
      // Convert spiderInfo to JSON
      String spiderInfoJSON = new com.google.gson.Gson().toJson(spiderInfo);
      
      // Prepare form data
      String formData = "spider_name=" + URLEncoder.encode("amazon.com", StandardCharsets.UTF_8) + 
        "&individual_params=" + URLEncoder.encode(spiderInfoJSON, StandardCharsets.UTF_8) +
        "&file_name=" + URLEncoder.encode("{{TasksID}}", StandardCharsets.UTF_8);

      // Create and configure the connection
      HttpURLConnection connection = (HttpURLConnection) new URL(targetURL).openConnection();
      connection.setRequestMethod("POST");
      connection.setRequestProperty("Authorization", "Bearer Token-ID");
      connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
      connection.setDoOutput(true);

      // Send the request
      try (OutputStream os = connection.getOutputStream()) {
        byte[] input = formData.getBytes(StandardCharsets.UTF_8);
        os.write(input, 0, input.length);
      }

      // Get the response
      int statusCode = connection.getResponseCode();
      String responseBody;
      try (InputStream is = connection.getInputStream();
        BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
        StringBuilder response = new StringBuilder();
        String responseLine;
        while ((responseLine = br.readLine()) != null) {
          response.append(responseLine.trim());
        }
        responseBody = response.toString();
      }

      System.out.println("Status Code: " + statusCode);
      System.out.println("Response Body: " + responseBody);

    } catch (Exception e) {
      System.err.println("Error: " + e.getMessage());
      e.printStackTrace();
    }
  }
}

Ruby

require 'json'
require 'net/http'
require 'uri'

begin
  target_url = URI.parse('http://scraperapi.thordata.com/builder')

  individual_params = {
    spider_parameters: [
      {
        url: 'https://www.amazon.com/HISDERN-Checkered-Handkerchief-Classic-Necktie/dp/B0BRXPR726',
        zip_code: '25662'
      }
    ],
    spider_id: 'amazon_product_by-url',
    spider_errors: true
  }

  individual_params_json = individual_params.to_json
  
  form_data = URI.encode_www_form(
    spider_name: 'amazon.com',
    individual_params: individual_params_json,
    file_name: '{{TasksID}}'
  )

  http = Net::HTTP.new(target_url.host, target_url.port)
  request = Net::HTTP::Post.new(target_url.path)
  request['Authorization'] = 'Bearer Token-ID'
  request['Content-Type'] = 'application/x-www-form-urlencoded'
  request.body = form_data

  response = http.request(request)

  puts "Status Code: #{response.code}"
  puts "Response Body: #{response.body}"

rescue JSON::GeneratorError => e
  puts "Error generating JSON: #{e.message}"
rescue URI::InvalidURIError => e
  puts "Invalid URL: #{e.message}"
rescue Net::HTTPError => e
  puts "HTTP Error: #{e.message}"
rescue StandardError => e
  puts "Error: #{e.message}"
end

After sending the request, the system will return the output in JSON or CSV format. You can view it in the “Tasks” list.

If you need further assistance, please contact us via email at [email protected].