Send Your First Request
Send your first request using Thordata's Universal Scraping API. Before you start, you will need an API Token.
You can get a free trial on the [Pricing] page.
Then, copy your credentials under [API Playground] > "Token".
Code example: After obtaining your API credentials, use the following code to send your first request:
curl -X POST https://universalapi.thordata.com/request \
-H "Authorization: Bearer token" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "url=https://www.google.com" \
-d "type=html" \
-d "js_render=True" \
-d "header=False"import http.client
from urllib.parse import urlencode
conn = http.client.HTTPSConnection("universalapi.thordata.com")
payload = {
"url": "https://www.google.com",
"type": "html",
"js_render": "True",
"header": "False"
}
form_data = urlencode(payload)
headers = {
'Authorization': "Bearer token",
'content-type': "application/x-www-form-urlencoded"
}
conn.request("POST", "/request", form_data, headers)
res = conn.getresponse()
data = res.read()
print(f"Status: {res.status} {res.reason}")
print(data.decode("utf-8"))using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Web;
class Program
{
static async Task Main(string[] args)
{
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri(" https://universalapi.thordata.com/request"),
Headers =
{
{ "Authorization", "Bearer token" },
},
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
{"url", "https://www.google.com"},
{"type", "html"},
{"js_render", "True"},
{"header", "False"}
})
};
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);
}
}
} package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"strings"
"time"
)
func main() {
var apiKey = "token"
var targetURL = "https://universalapi.thordata.com/request"
formData := url.Values{
"url": {"https://www.google.com"},
"type": {"html"},
"js_render": {"True"},
"header": {"False"},
}
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode()))
if err != nil {
log.Fatal("Create request failed:", err)
}
req.Header = http.Header{
"Authorization": {"Bearer " + apiKey},
"Content-Type": {"application/x-www-form-urlencoded"},
}
res, err := client.Do(req)
if err != nil {
log.Fatal("Failed to send request:", err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Fatal("Failed to read response:", err)
}
fmt.Printf("Status: %d Content:%s", res.StatusCode, body)
}const https = require("https");
const querystring = require("querystring");
const options = {
method: "POST",
hostname: "universalapi.thordata.com",
path: "/request",
headers: {
"Authorization": "Bearer token",
"content-type": "application/x-www-form-urlencoded"
}
};
const formData = {
url: "https://www.google.com",
type: "html",
js_render: "True",
header: "False"
};
const formDataString = querystring.stringify(formData);
options.headers["Content-Length"] = formDataString.length;
const req = https.request(options, (res) => {
const chunks = [];
res.on("data", (chunk) => chunks.push(chunk));
res.on("end", () => {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.write(formDataString);
req.end();<?php
$data = [
'url'=> 'https://www.google.com',
'type'=> 'html',
'js_render'=> 'True',
'header'=> 'False'
];
$queryString = http_build_query($data);
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://universalapi.thordata.com/request",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => $queryString,
CURLOPT_HTTPHEADER => [
"Authorization: Bearer token",
"content-type: application/x-www-form-urlencoded"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://universalapi.thordata.com/request")
form_data = {
"url"=> "https://www.google.com",
"type"=> "html",
"js_render"=> "True",
"header"=> "False"
}
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer token'
request["content-type"] = 'application/x-www-form-urlencoded'
request.set_form_data(form_data)
response = http.request(request)
puts response.bodyimport java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.stream.Collectors;
public class Main {
public static void main(String[] args) throws Exception {
String formData = buildFormData(
"url", "https://www.google.com",
"type", "html",
"js_render", "True",
"header", "False"
);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://universalapi.thordata.com/request"))
.header("Authorization", "Bearer token")
.header("content-type", "application/x-www-form-urlencoded")
.POST(HttpRequest.BodyPublishers.ofString(formData))
.build();
HttpResponse<String> response = HttpClient.newHttpClient()
.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());
}
private static String buildFormData(String... keyValuePairs) {
if (keyValuePairs.length % 2 != 0) {
throw new IllegalArgumentException("Parameters must be key-value pairs.");
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < keyValuePairs.length; i += 2) {
if (i > 0) sb.append("&");
sb.append(keyValuePairs[i])
.append("=")
.append(URLEncoder.encode(keyValuePairs[i+1], StandardCharsets.UTF_8));
}
return sb.toString();
}
}After sending the request, it will return output results in HTML format:
<!DOCTYPE html>
<html itemscope="" itemtype="http://schema.org/WebPage" lang="en">
<head>
<meta charset="UTF-8">
<meta content="origin" name="referrer">
<link href="//www.gstatic.com/images/branding/searchlogo/ico/favicon.ico" rel="icon">
<meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image">
<style>
@font-face {
font-family: 'Google Sans';
font-style: normal;
font-weight: 400 700;
font-display: optional;
src: url(//fonts.gstatic.com/s/googlesans/v29/4UaGrENHsxJlGDuGo1OIlL3Owp4.woff2) format('woff2');
unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
@font-face {
font-family: 'Roboto';
font-style: normal;
font-weight: 400;
font-display: optional;
src: url(//fonts.gstatic.com/s/roboto/v18/KFOmCnqEu92Fr1Mu4mxK.woff2) format('woff2');
unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
@font-face {
font-family: 'Roboto';
font-style: normal;
font-weight: 700;
font-display: optional;
src: url(//fonts.gstatic.com/s/roboto/v18/KFOlCnqEu92Fr1MmWUlfBBc4.woff2) format('woff2');
unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;
}
</style>
<script nonce="">
(function(){
var w=["Google Sans",[400,500,700],"Roboto",[400,700]];
(function(){
for(var a=0;a<w.length;a+=2)
for(var d=w[a],e=w[a+1],b=0,c=void 0;c=e[b];++b)
document.fonts.load(c+" 10pt "+d).catch(function(){})
...If you need further assistance, contact us at [email protected].
Last updated
Was this helpful?