Create an ingest pipeline
curl --request POST \
--url https://api.example.com/api/ingest/pipelines \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"source": {
"kind": "file",
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"s3": "<string>",
"options": {}
},
"destination": {
"schema": "<string>",
"table": "<string>",
"mode": "append",
"merge_keys": [
"<string>"
],
"schedule": "<string>"
},
"columns": [
{
"name": "<string>",
"load_as": "<string>",
"rename": "<string>",
"drop": false,
"encrypt": false
}
],
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"classify": false,
"suggest_checks": false
}
}
'import requests
url = "https://api.example.com/api/ingest/pipelines"
payload = {
"name": "<string>",
"source": {
"kind": "file",
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"s3": "<string>",
"options": {}
},
"destination": {
"schema": "<string>",
"table": "<string>",
"mode": "append",
"merge_keys": ["<string>"],
"schedule": "<string>"
},
"columns": [
{
"name": "<string>",
"load_as": "<string>",
"rename": "<string>",
"drop": False,
"encrypt": False
}
],
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": ["<string>"],
"classify": False,
"suggest_checks": False
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
source: {
kind: 'file',
file_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
url: '<string>',
s3: '<string>',
options: {}
},
destination: {
schema: '<string>',
table: '<string>',
mode: 'append',
merge_keys: ['<string>'],
schedule: '<string>'
},
columns: [
{
name: '<string>',
load_as: '<string>',
rename: '<string>',
drop: false,
encrypt: false
}
],
governance: {
owner: '<string>',
domain: '<string>',
tags: ['<string>'],
classify: false,
suggest_checks: false
}
})
};
fetch('https://api.example.com/api/ingest/pipelines', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/ingest/pipelines",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'source' => [
'kind' => 'file',
'file_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'url' => '<string>',
's3' => '<string>',
'options' => [
]
],
'destination' => [
'schema' => '<string>',
'table' => '<string>',
'mode' => 'append',
'merge_keys' => [
'<string>'
],
'schedule' => '<string>'
],
'columns' => [
[
'name' => '<string>',
'load_as' => '<string>',
'rename' => '<string>',
'drop' => false,
'encrypt' => false
]
],
'governance' => [
'owner' => '<string>',
'domain' => '<string>',
'tags' => [
'<string>'
],
'classify' => false,
'suggest_checks' => false
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/ingest/pipelines"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/ingest/pipelines")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/ingest/pipelines")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"name": "<string>",
"source_kind": "file",
"source_summary": "<string>",
"target_table": "<string>",
"mode": "append",
"schedule_label": "<string>",
"status": "healthy",
"owner": "<string>",
"can_edit": true,
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"classify": false,
"suggest_checks": false
},
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"schedule": "<string>",
"last_run": {
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"trigger": "schedule",
"status": "queued",
"phase": "reading",
"started_at": "2023-11-07T05:31:56Z",
"finished_at": "2023-11-07T05:31:56Z",
"duration_s": 123,
"rows_read": 123,
"rows_written": 123,
"checks_passed": 123,
"checks_total": 123,
"error_kind": "schema_drift",
"error": "<string>",
"schema_change": {
"added": [
"<string>"
],
"changed": [
"<string>"
]
},
"governance": {}
},
"rows_7d": 0,
"merge_keys": [
"<string>"
],
"job_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"airflow_url": "<string>",
"spec": {},
"latest_error": {
"kind": "schema_drift",
"message": "<string>",
"schema_change": {
"added": [
"<string>"
],
"changed": [
"<string>"
]
}
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Ingest pipelines
Create an ingest pipeline
Validates, creates the pipeline, schedules its DAG and links the job.
Without a schedule it runs at once, and the first run is listed as queued.
POST
/
api
/
ingest
/
pipelines
Create an ingest pipeline
curl --request POST \
--url https://api.example.com/api/ingest/pipelines \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"source": {
"kind": "file",
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"s3": "<string>",
"options": {}
},
"destination": {
"schema": "<string>",
"table": "<string>",
"mode": "append",
"merge_keys": [
"<string>"
],
"schedule": "<string>"
},
"columns": [
{
"name": "<string>",
"load_as": "<string>",
"rename": "<string>",
"drop": false,
"encrypt": false
}
],
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"classify": false,
"suggest_checks": false
}
}
'import requests
url = "https://api.example.com/api/ingest/pipelines"
payload = {
"name": "<string>",
"source": {
"kind": "file",
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"s3": "<string>",
"options": {}
},
"destination": {
"schema": "<string>",
"table": "<string>",
"mode": "append",
"merge_keys": ["<string>"],
"schedule": "<string>"
},
"columns": [
{
"name": "<string>",
"load_as": "<string>",
"rename": "<string>",
"drop": False,
"encrypt": False
}
],
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": ["<string>"],
"classify": False,
"suggest_checks": False
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
source: {
kind: 'file',
file_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
url: '<string>',
s3: '<string>',
options: {}
},
destination: {
schema: '<string>',
table: '<string>',
mode: 'append',
merge_keys: ['<string>'],
schedule: '<string>'
},
columns: [
{
name: '<string>',
load_as: '<string>',
rename: '<string>',
drop: false,
encrypt: false
}
],
governance: {
owner: '<string>',
domain: '<string>',
tags: ['<string>'],
classify: false,
suggest_checks: false
}
})
};
fetch('https://api.example.com/api/ingest/pipelines', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/ingest/pipelines",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'source' => [
'kind' => 'file',
'file_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'url' => '<string>',
's3' => '<string>',
'options' => [
]
],
'destination' => [
'schema' => '<string>',
'table' => '<string>',
'mode' => 'append',
'merge_keys' => [
'<string>'
],
'schedule' => '<string>'
],
'columns' => [
[
'name' => '<string>',
'load_as' => '<string>',
'rename' => '<string>',
'drop' => false,
'encrypt' => false
]
],
'governance' => [
'owner' => '<string>',
'domain' => '<string>',
'tags' => [
'<string>'
],
'classify' => false,
'suggest_checks' => false
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/ingest/pipelines"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/ingest/pipelines")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/ingest/pipelines")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"source\": {\n \"kind\": \"file\",\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"url\": \"<string>\",\n \"s3\": \"<string>\",\n \"options\": {}\n },\n \"destination\": {\n \"schema\": \"<string>\",\n \"table\": \"<string>\",\n \"mode\": \"append\",\n \"merge_keys\": [\n \"<string>\"\n ],\n \"schedule\": \"<string>\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"load_as\": \"<string>\",\n \"rename\": \"<string>\",\n \"drop\": false,\n \"encrypt\": false\n }\n ],\n \"governance\": {\n \"owner\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"classify\": false,\n \"suggest_checks\": false\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"name": "<string>",
"source_kind": "file",
"source_summary": "<string>",
"target_table": "<string>",
"mode": "append",
"schedule_label": "<string>",
"status": "healthy",
"owner": "<string>",
"can_edit": true,
"governance": {
"owner": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"classify": false,
"suggest_checks": false
},
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"schedule": "<string>",
"last_run": {
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"trigger": "schedule",
"status": "queued",
"phase": "reading",
"started_at": "2023-11-07T05:31:56Z",
"finished_at": "2023-11-07T05:31:56Z",
"duration_s": 123,
"rows_read": 123,
"rows_written": 123,
"checks_passed": 123,
"checks_total": 123,
"error_kind": "schema_drift",
"error": "<string>",
"schema_change": {
"added": [
"<string>"
],
"changed": [
"<string>"
]
},
"governance": {}
},
"rows_7d": 0,
"merge_keys": [
"<string>"
],
"job_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"airflow_url": "<string>",
"spec": {},
"latest_error": {
"kind": "schema_drift",
"message": "<string>",
"schema_change": {
"added": [
"<string>"
],
"changed": [
"<string>"
]
}
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
OAuth2AuthorizationCodeBearerAPIKeyHeader
The access token received from the authorization server in the OAuth 2.0 flow.
Body
application/json
- FileSource
- DatabaseSource
- LakehouseSource
- SaasSource
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Maximum array length:
500Show child attributes
Show child attributes
Show child attributes
Show child attributes
Response
Successful Response
Available options:
file, database, lakehouse, saas Available options:
append, overwrite, merge Available options:
healthy, failed, running, stale, paused, new Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?

