Get all crawl requests for a dataset
curl --request GET \
--url https://api.trieve.ai/api/crawl \
--header 'Authorization: <api-key>' \
--header 'TR-Dataset: <tr-dataset>'import requests
url = "https://api.trieve.ai/api/crawl"
headers = {
"TR-Dataset": "<tr-dataset>",
"Authorization": "<api-key>"
}
response = requests.get(url, headers=headers)
print(response.text)const options = {
method: 'GET',
headers: {'TR-Dataset': '<tr-dataset>', Authorization: '<api-key>'}
};
fetch('https://api.trieve.ai/api/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.trieve.ai/api/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: <api-key>",
"TR-Dataset: <tr-dataset>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.trieve.ai/api/crawl"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("TR-Dataset", "<tr-dataset>")
req.Header.Add("Authorization", "<api-key>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.trieve.ai/api/crawl")
.header("TR-Dataset", "<tr-dataset>")
.header("Authorization", "<api-key>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.trieve.ai/api/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["TR-Dataset"] = '<tr-dataset>'
request["Authorization"] = '<api-key>'
response = http.request(request)
puts response.read_body[
{
"attempt_number": 123,
"crawl_options": {
"crawl_options": {
"allow_external_links": false,
"boost_titles": true,
"exclude_tags": [
"#ad",
"#footer",
"header",
"head",
"navbar",
"footer",
"aside",
"nav",
"form"
],
"heading_remove_strings": [
"Advertisement",
"Sponsored"
],
"ignore_sitemap": true,
"include_tags": [],
"interval": "daily",
"limit": 50,
"site_url": "nedzo.ai"
}
},
"crawl_type": "firecrawl",
"created_at": "2023-11-07T05:31:56Z",
"dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"scrape_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"status": "Pending",
"url": "<string>",
"interval": "<string>",
"next_crawl_at": "2023-11-07T05:31:56Z"
}
]{
"message": "Bad Request"
}Crawl
Get all crawl requests for a dataset
This endpoint is used to get all crawl requests for a dataset.
GET
/
api
/
crawl
Get all crawl requests for a dataset
curl --request GET \
--url https://api.trieve.ai/api/crawl \
--header 'Authorization: <api-key>' \
--header 'TR-Dataset: <tr-dataset>'import requests
url = "https://api.trieve.ai/api/crawl"
headers = {
"TR-Dataset": "<tr-dataset>",
"Authorization": "<api-key>"
}
response = requests.get(url, headers=headers)
print(response.text)const options = {
method: 'GET',
headers: {'TR-Dataset': '<tr-dataset>', Authorization: '<api-key>'}
};
fetch('https://api.trieve.ai/api/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.trieve.ai/api/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: <api-key>",
"TR-Dataset: <tr-dataset>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.trieve.ai/api/crawl"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("TR-Dataset", "<tr-dataset>")
req.Header.Add("Authorization", "<api-key>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.trieve.ai/api/crawl")
.header("TR-Dataset", "<tr-dataset>")
.header("Authorization", "<api-key>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.trieve.ai/api/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["TR-Dataset"] = '<tr-dataset>'
request["Authorization"] = '<api-key>'
response = http.request(request)
puts response.read_body[
{
"attempt_number": 123,
"crawl_options": {
"crawl_options": {
"allow_external_links": false,
"boost_titles": true,
"exclude_tags": [
"#ad",
"#footer",
"header",
"head",
"navbar",
"footer",
"aside",
"nav",
"form"
],
"heading_remove_strings": [
"Advertisement",
"Sponsored"
],
"ignore_sitemap": true,
"include_tags": [],
"interval": "daily",
"limit": 50,
"site_url": "nedzo.ai"
}
},
"crawl_type": "firecrawl",
"created_at": "2023-11-07T05:31:56Z",
"dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"scrape_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"status": "Pending",
"url": "<string>",
"interval": "<string>",
"next_crawl_at": "2023-11-07T05:31:56Z"
}
]{
"message": "Bad Request"
}Authorizations
Headers
The dataset id to use for the request
Query Parameters
The page number to retrieve
The number of items to retrieve per page
Response
Crawl requests retrieved successfully
Options for setting up the crawl which will populate the dataset.
Show child attributes
Show child attributes
Example:
{
"crawl_options": {
"allow_external_links": false,
"boost_titles": true,
"exclude_tags": [
"#ad",
"#footer",
"header",
"head",
"navbar",
"footer",
"aside",
"nav",
"form"
],
"heading_remove_strings": ["Advertisement", "Sponsored"],
"ignore_sitemap": true,
"include_tags": [],
"interval": "daily",
"limit": 50,
"site_url": "nedzo.ai"
}
}
Available options:
firecrawl, openapi, shopify, youtube status
Option 1 · enum<string>Option 2 · objectOption 3 · enum<string>Option 4 · enum<string>
required
Available options:
Pending Was this page helpful?