Create a new crawl request

This endpoint is used to create a new crawl request for a dataset. The request payload should contain the crawl options to use for the crawl.

POST /api/crawl

Create a new crawl request

cURL

curl --request POST \
  --url https://api.trieve.ai/api/crawl \
  --header 'Authorization: <api-key>' \
  --header 'Content-Type: application/json' \
  --header 'TR-Dataset: <tr-dataset>' \
  --data '
{
  "crawl_options": {
    "crawl_options": {
      "allow_external_links": false,
      "boost_titles": true,
      "exclude_tags": [
        "#ad",
        "#footer",
        "header",
        "head",
        "navbar",
        "footer",
        "aside",
        "nav",
        "form"
      ],
      "heading_remove_strings": [
        "Advertisement",
        "Sponsored"
      ],
      "ignore_sitemap": true,
      "include_tags": [],
      "interval": "daily",
      "limit": 50,
      "site_url": "nedzo.ai"
    }
  }
}
'

import requests

url = "https://api.trieve.ai/api/crawl"

payload = { "crawl_options": { "crawl_options": {
            "allow_external_links": False,
            "boost_titles": True,
            "exclude_tags": ["#ad", "#footer", "header", "head", "navbar", "footer", "aside", "nav", "form"],
            "heading_remove_strings": ["Advertisement", "Sponsored"],
            "ignore_sitemap": True,
            "include_tags": [],
            "interval": "daily",
            "limit": 50,
            "site_url": "nedzo.ai"
        } } }
headers = {
    "TR-Dataset": "<tr-dataset>",
    "Authorization": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'TR-Dataset': '<tr-dataset>',
    Authorization: '<api-key>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    crawl_options: {
      crawl_options: {
        allow_external_links: false,
        boost_titles: true,
        exclude_tags: ['#ad', '#footer', 'header', 'head', 'navbar', 'footer', 'aside', 'nav', 'form'],
        heading_remove_strings: ['Advertisement', 'Sponsored'],
        ignore_sitemap: true,
        include_tags: [],
        interval: 'daily',
        limit: 50,
        site_url: 'nedzo.ai'
      }
    }
  })
};

fetch('https://api.trieve.ai/api/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.trieve.ai/api/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'crawl_options' => [
        'crawl_options' => [
                'allow_external_links' => false,
                'boost_titles' => true,
                'exclude_tags' => [
                                '#ad',
                                '#footer',
                                'header',
                                'head',
                                'navbar',
                                'footer',
                                'aside',
                                'nav',
                                'form'
                ],
                'heading_remove_strings' => [
                                'Advertisement',
                                'Sponsored'
                ],
                'ignore_sitemap' => true,
                'include_tags' => [
                                
                ],
                'interval' => 'daily',
                'limit' => 50,
                'site_url' => 'nedzo.ai'
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <api-key>",
    "Content-Type: application/json",
    "TR-Dataset: <tr-dataset>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.trieve.ai/api/crawl"

	payload := strings.NewReader("{\n  \"crawl_options\": {\n    \"crawl_options\": {\n      \"allow_external_links\": false,\n      \"boost_titles\": true,\n      \"exclude_tags\": [\n        \"#ad\",\n        \"#footer\",\n        \"header\",\n        \"head\",\n        \"navbar\",\n        \"footer\",\n        \"aside\",\n        \"nav\",\n        \"form\"\n      ],\n      \"heading_remove_strings\": [\n        \"Advertisement\",\n        \"Sponsored\"\n      ],\n      \"ignore_sitemap\": true,\n      \"include_tags\": [],\n      \"interval\": \"daily\",\n      \"limit\": 50,\n      \"site_url\": \"nedzo.ai\"\n    }\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("TR-Dataset", "<tr-dataset>")
	req.Header.Add("Authorization", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.trieve.ai/api/crawl")
  .header("TR-Dataset", "<tr-dataset>")
  .header("Authorization", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"crawl_options\": {\n    \"crawl_options\": {\n      \"allow_external_links\": false,\n      \"boost_titles\": true,\n      \"exclude_tags\": [\n        \"#ad\",\n        \"#footer\",\n        \"header\",\n        \"head\",\n        \"navbar\",\n        \"footer\",\n        \"aside\",\n        \"nav\",\n        \"form\"\n      ],\n      \"heading_remove_strings\": [\n        \"Advertisement\",\n        \"Sponsored\"\n      ],\n      \"ignore_sitemap\": true,\n      \"include_tags\": [],\n      \"interval\": \"daily\",\n      \"limit\": 50,\n      \"site_url\": \"nedzo.ai\"\n    }\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.trieve.ai/api/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["TR-Dataset"] = '<tr-dataset>'
request["Authorization"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"crawl_options\": {\n    \"crawl_options\": {\n      \"allow_external_links\": false,\n      \"boost_titles\": true,\n      \"exclude_tags\": [\n        \"#ad\",\n        \"#footer\",\n        \"header\",\n        \"head\",\n        \"navbar\",\n        \"footer\",\n        \"aside\",\n        \"nav\",\n        \"form\"\n      ],\n      \"heading_remove_strings\": [\n        \"Advertisement\",\n        \"Sponsored\"\n      ],\n      \"ignore_sitemap\": true,\n      \"include_tags\": [],\n      \"interval\": \"daily\",\n      \"limit\": 50,\n      \"site_url\": \"nedzo.ai\"\n    }\n  }\n}"

response = http.request(request)
puts response.read_body

200

{
  "attempt_number": 123,
  "crawl_options": {
    "crawl_options": {
      "allow_external_links": false,
      "boost_titles": true,
      "exclude_tags": [
        "#ad",
        "#footer",
        "header",
        "head",
        "navbar",
        "footer",
        "aside",
        "nav",
        "form"
      ],
      "heading_remove_strings": [
        "Advertisement",
        "Sponsored"
      ],
      "ignore_sitemap": true,
      "include_tags": [],
      "interval": "daily",
      "limit": 50,
      "site_url": "nedzo.ai"
    }
  },
  "created_at": "2023-11-07T05:31:56Z",
  "dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "scrape_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "status": "Pending",
  "url": "<string>",
  "interval": "<string>",
  "next_crawl_at": "2023-11-07T05:31:56Z"
}

{
  "message": "Bad Request"
}

Authorizations

Authorization · string · header · required

Headers

TR-Dataset · string<uuid> · required

The dataset id to use for the request

Body

application/json

JSON request payload to create a new crawl

crawl_options · object · required

Options for setting up the crawl which will populate the dataset.

Show child attributes

Example:

{
  "crawl_options": {
    "allow_external_links": false,
    "boost_titles": true,
    "exclude_tags": [
      "#ad",
      "#footer",
      "header",
      "head",
      "navbar",
      "footer",
      "aside",
      "nav",
      "form"
    ],
    "heading_remove_strings": ["Advertisement", "Sponsored"],
    "ignore_sitemap": true,
    "include_tags": [],
    "interval": "daily",
    "limit": 50,
    "site_url": "nedzo.ai"
  }
}

Response

200

application/json

Crawl created successfully

attempt_number · integer<int32> · required

crawl_options · object · required

Options for setting up the crawl which will populate the dataset.

Show child attributes

Example:

{
  "crawl_options": {
    "allow_external_links": false,
    "boost_titles": true,
    "exclude_tags": [
      "#ad",
      "#footer",
      "header",
      "head",
      "navbar",
      "footer",
      "aside",
      "nav",
      "form"
    ],
    "heading_remove_strings": ["Advertisement", "Sponsored"],
    "ignore_sitemap": true,
    "include_tags": [],
    "interval": "daily",
    "limit": 50,
    "site_url": "nedzo.ai"
  }
}

crawl_type · enum<string> · required

Available options: firecrawl, openapi, shopify, youtube

created_at · string<date-time> · required

dataset_id · string<uuid> · required

id · string<uuid> · required

scrape_id · string<uuid> · required

status · Option 1 · enum<string> Option 2 · object Option 3 · enum<string> Option 4 · enum<string> · required

Available options: Pending

url · string · required

interval · string | null

next_crawl_at · string<date-time> | null