Send Your First Request

Send your first request using Thordata's Universal Scraping API. Before you start, you will need an API Token.

1. You can get a free trial on the [Pricing] page.

2. Then, copy your credentials under [API Playground] > "Token".

Code example: After obtaining your API credentials, use the following code to send your first request:

cURL

curl -X POST https://universalapi.thordata.com/request \
 -H "Authorization: Bearer token" \
 -H "Content-Type: application/x-www-form-urlencoded" \
 -d "url=https://www.google.com" \
 -d "type=html" \
 -d "js_render=True" \
 -d "header=False"

Python

import http.client
from urllib.parse import urlencode

conn = http.client.HTTPSConnection("universalapi.thordata.com")

payload = {
    "url": "https://www.google.com",
    "type": "html",
    "js_render": "True",
    "header": "False"
}

form_data = urlencode(payload)

headers = {
    'Authorization': "Bearer token",
    'content-type': "application/x-www-form-urlencoded"
}

conn.request("POST", "/request", form_data, headers)

res = conn.getresponse()
data = res.read()

print(f"Status: {res.status} {res.reason}")
print(data.decode("utf-8"))

C#

using System; 
using System.Net.Http; 
using System.Net.Http.Headers; 
using System.Collections.Generic; 
using System.Threading.Tasks; 
using System.Web; 
class Program 
{ 
    static async Task Main(string[] args) 
    {  
        var client = new HttpClient(); 
        var request = new HttpRequestMessage 
        { 
            Method = HttpMethod.Post, 
            RequestUri = new Uri(" https://universalapi.thordata.com/request"), 
            Headers = 
            { 
                { "Authorization", "Bearer token" },
            }, 
            Content = new FormUrlEncodedContent(new Dictionary<string, string> 
            {
            {"url", "https://www.google.com"},
            {"type", "html"},
            {"js_render", "True"},
            {"header", "False"}
            })
        }; 
        
        request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded"); 
        using (var response = await client.SendAsync(request)) 
        { 
            response.EnsureSuccessStatusCode(); 
            var body = await response.Content.ReadAsStringAsync(); 
            Console.WriteLine(body); 
        } 
    } 
} 

Go

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func main() {
	
	var apiKey = "token"
	var targetURL = "https://universalapi.thordata.com/request"
	
	formData := url.Values{
    "url": {"https://www.google.com"},
    "type": {"html"},
    "js_render": {"True"},
        "header": {"False"},
    }
	
	client := &http.Client{Timeout: 30 * time.Second}
	
	req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode()))
	if err != nil {
		log.Fatal("Create request failed:", err)
	}
	
	req.Header = http.Header{
		"Authorization": {"Bearer " + apiKey},
		"Content-Type":  {"application/x-www-form-urlencoded"},
	}
	
	res, err := client.Do(req)
	if err != nil {
		log.Fatal("Failed to send request:", err)
	}
	defer res.Body.Close()
	
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal("Failed to read response:", err)
	}
	
	fmt.Printf("Status: %d Content:%s", res.StatusCode, body)
}

Node.js

const https = require("https");
const querystring = require("querystring"); 

const options = {
    method: "POST",
    hostname: "universalapi.thordata.com",
    path: "/request",
    headers: {
        "Authorization": "Bearer token",
        "content-type": "application/x-www-form-urlencoded" 
    }
};
const formData = {
    url: "https://www.google.com",
    type: "html",
    js_render: "True",
    header: "False"
};

const formDataString = querystring.stringify(formData);
options.headers["Content-Length"] = formDataString.length;

const req = https.request(options, (res) => {
    const chunks = [];
    res.on("data", (chunk) => chunks.push(chunk));
    res.on("end", () => {
        const body = Buffer.concat(chunks);
        console.log(body.toString());
    });
});

req.write(formDataString);
req.end();

PHP

<?php
$data = [
    'url'=> 'https://www.google.com',
    'type'=> 'html',
    'js_render'=> 'True',
    'header'=> 'False'
];

$queryString = http_build_query($data);

$curl = curl_init();
curl_setopt_array($curl, [
    CURLOPT_URL => "https://universalapi.thordata.com/request",
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_ENCODING => "",
    CURLOPT_MAXREDIRS => 10,
    CURLOPT_TIMEOUT => 30,
    CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
    CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => $queryString, 
CURLOPT_HTTPHEADER => [
        "Authorization: Bearer token",
        "content-type: application/x-www-form-urlencoded" 
    ],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
    echo "cURL Error #:" . $err;
} else {
    echo $response;
}

Ruby

require 'uri'
require 'net/http'
require 'openssl'

url = URI("https://universalapi.thordata.com/request")

form_data = {
    "url"=> "https://www.google.com",
    "type"=> "html",
    "js_render"=> "True",
    "header"=> "False"
}

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Post.new(url)

request["Authorization"] = 'Bearer token'
request["content-type"] = 'application/x-www-form-urlencoded' 

request.set_form_data(form_data)

response = http.request(request)
puts response.body

Java

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.stream.Collectors;

public class Main {
    public static void main(String[] args) throws Exception {
        String formData = buildFormData(
            "url", "https://www.google.com",
            "type", "html",
            "js_render", "True",
            "header", "False"
    );
        
        HttpRequest request = HttpRequest.newBuilder()
                .uri(URI.create("https://universalapi.thordata.com/request"))
                .header("Authorization", "Bearer token")
                .header("content-type", "application/x-www-form-urlencoded")
                .POST(HttpRequest.BodyPublishers.ofString(formData))
                .build();
        
        HttpResponse<String> response = HttpClient.newHttpClient()
                .send(request, HttpResponse.BodyHandlers.ofString());
        
        System.out.println(response.body());
    }
    
    
    private static String buildFormData(String... keyValuePairs) {
        if (keyValuePairs.length % 2 != 0) {
            throw new IllegalArgumentException("Parameters must be key-value pairs.");
        }
        
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < keyValuePairs.length; i += 2) {
            if (i > 0) sb.append("&");
            sb.append(keyValuePairs[i])
              .append("=")
              .append(URLEncoder.encode(keyValuePairs[i+1], StandardCharsets.UTF_8));
        }
        return sb.toString();
    }
}

After sending the request, it will return output results in HTML format:

Output

<!DOCTYPE html>
<html itemscope="" itemtype="http://schema.org/WebPage" lang="en">
<head>
<meta charset="UTF-8">
<meta content="origin" name="referrer">
<link href="//www.gstatic.com/images/branding/searchlogo/ico/favicon.ico" rel="icon">
<meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image">

<style>
@font-face {
  font-family: 'Google Sans';
  font-style: normal;
  font-weight: 400 700;
  font-display: optional;
  src: url(//fonts.gstatic.com/s/googlesans/v29/4UaGrENHsxJlGDuGo1OIlL3Owp4.woff2) format('woff2');
  unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
@font-face {
  font-family: 'Roboto';
  font-style: normal;
  font-weight: 400;
  font-display: optional;
  src: url(//fonts.gstatic.com/s/roboto/v18/KFOmCnqEu92Fr1Mu4mxK.woff2) format('woff2');
  unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
@font-face {
  font-family: 'Roboto';
  font-style: normal;
  font-weight: 700;
  font-display: optional;
  src: url(//fonts.gstatic.com/s/roboto/v18/KFOlCnqEu92Fr1MmWUlfBBc4.woff2) format('woff2');
  unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
</style>

<script nonce="">
(function(){
  var w=["Google Sans",[400,500,700],"Roboto",[400,700]];
  (function(){
    for(var a=0;a<w.length;a+=2)
      for(var d=w[a],e=w[a+1],b=0,c=void 0;c=e[b];++b)
        document.fonts.load(c+" 10pt "+d).catch(function(){})
 ...

If you need further assistance, contact us at [email protected].