Ingest
curl --request POST \
--url https://api.example.com/api/dataingest/submit \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"schema_name": "<string>",
"table": "<string>",
"schedule": "<string>",
"merge": "<string>",
"column_transformations": [
{
"column": "<string>",
"stream": "<string>",
"new_name": "<string>",
"encryption_key_name": "<string>"
}
],
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"file_path": "<string>",
"file_format": "<string>",
"file_read_options": {},
"jdbc_url": "<string>",
"jdbc_username": "<string>",
"jdbc_password": "<string>",
"jdbc_schema": "<string>",
"jdbc_table": "<string>",
"jdbc_query": "<string>",
"lakehouse_sourceschema": "<string>",
"lakehouse_sourcetable": "<string>",
"lakehouse_conditions": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"owner_id": "<string>"
}
'import requests
url = "https://api.example.com/api/dataingest/submit"
payload = {
"name": "<string>",
"schema_name": "<string>",
"table": "<string>",
"schedule": "<string>",
"merge": "<string>",
"column_transformations": [
{
"column": "<string>",
"stream": "<string>",
"new_name": "<string>",
"encryption_key_name": "<string>"
}
],
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"file_path": "<string>",
"file_format": "<string>",
"file_read_options": {},
"jdbc_url": "<string>",
"jdbc_username": "<string>",
"jdbc_password": "<string>",
"jdbc_schema": "<string>",
"jdbc_table": "<string>",
"jdbc_query": "<string>",
"lakehouse_sourceschema": "<string>",
"lakehouse_sourcetable": "<string>",
"lakehouse_conditions": "<string>",
"domain": "<string>",
"tags": ["<string>"],
"owner_id": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
schema_name: '<string>',
table: '<string>',
schedule: '<string>',
merge: '<string>',
column_transformations: [
{
column: '<string>',
stream: '<string>',
new_name: '<string>',
encryption_key_name: '<string>'
}
],
file_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
file_path: '<string>',
file_format: '<string>',
file_read_options: {},
jdbc_url: '<string>',
jdbc_username: '<string>',
jdbc_password: '<string>',
jdbc_schema: '<string>',
jdbc_table: '<string>',
jdbc_query: '<string>',
lakehouse_sourceschema: '<string>',
lakehouse_sourcetable: '<string>',
lakehouse_conditions: '<string>',
domain: '<string>',
tags: ['<string>'],
owner_id: '<string>'
})
};
fetch('https://api.example.com/api/dataingest/submit', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/dataingest/submit",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'schema_name' => '<string>',
'table' => '<string>',
'schedule' => '<string>',
'merge' => '<string>',
'column_transformations' => [
[
'column' => '<string>',
'stream' => '<string>',
'new_name' => '<string>',
'encryption_key_name' => '<string>'
]
],
'file_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'file_path' => '<string>',
'file_format' => '<string>',
'file_read_options' => [
],
'jdbc_url' => '<string>',
'jdbc_username' => '<string>',
'jdbc_password' => '<string>',
'jdbc_schema' => '<string>',
'jdbc_table' => '<string>',
'jdbc_query' => '<string>',
'lakehouse_sourceschema' => '<string>',
'lakehouse_sourcetable' => '<string>',
'lakehouse_conditions' => '<string>',
'domain' => '<string>',
'tags' => [
'<string>'
],
'owner_id' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/dataingest/submit"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/dataingest/submit")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/dataingest/submit")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"flow_url": "<string>",
"job_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"error": "<string>"
}{
"error": "<string>",
"code": 500
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Data ingestion
Ingest
Submit a new data ingestion request to upload the required data into NexusOne. Returns the ingestion flow details.
POST
/
api
/
dataingest
/
submit
Ingest
curl --request POST \
--url https://api.example.com/api/dataingest/submit \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"schema_name": "<string>",
"table": "<string>",
"schedule": "<string>",
"merge": "<string>",
"column_transformations": [
{
"column": "<string>",
"stream": "<string>",
"new_name": "<string>",
"encryption_key_name": "<string>"
}
],
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"file_path": "<string>",
"file_format": "<string>",
"file_read_options": {},
"jdbc_url": "<string>",
"jdbc_username": "<string>",
"jdbc_password": "<string>",
"jdbc_schema": "<string>",
"jdbc_table": "<string>",
"jdbc_query": "<string>",
"lakehouse_sourceschema": "<string>",
"lakehouse_sourcetable": "<string>",
"lakehouse_conditions": "<string>",
"domain": "<string>",
"tags": [
"<string>"
],
"owner_id": "<string>"
}
'import requests
url = "https://api.example.com/api/dataingest/submit"
payload = {
"name": "<string>",
"schema_name": "<string>",
"table": "<string>",
"schedule": "<string>",
"merge": "<string>",
"column_transformations": [
{
"column": "<string>",
"stream": "<string>",
"new_name": "<string>",
"encryption_key_name": "<string>"
}
],
"file_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"file_path": "<string>",
"file_format": "<string>",
"file_read_options": {},
"jdbc_url": "<string>",
"jdbc_username": "<string>",
"jdbc_password": "<string>",
"jdbc_schema": "<string>",
"jdbc_table": "<string>",
"jdbc_query": "<string>",
"lakehouse_sourceschema": "<string>",
"lakehouse_sourcetable": "<string>",
"lakehouse_conditions": "<string>",
"domain": "<string>",
"tags": ["<string>"],
"owner_id": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
schema_name: '<string>',
table: '<string>',
schedule: '<string>',
merge: '<string>',
column_transformations: [
{
column: '<string>',
stream: '<string>',
new_name: '<string>',
encryption_key_name: '<string>'
}
],
file_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
file_path: '<string>',
file_format: '<string>',
file_read_options: {},
jdbc_url: '<string>',
jdbc_username: '<string>',
jdbc_password: '<string>',
jdbc_schema: '<string>',
jdbc_table: '<string>',
jdbc_query: '<string>',
lakehouse_sourceschema: '<string>',
lakehouse_sourcetable: '<string>',
lakehouse_conditions: '<string>',
domain: '<string>',
tags: ['<string>'],
owner_id: '<string>'
})
};
fetch('https://api.example.com/api/dataingest/submit', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/dataingest/submit",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'schema_name' => '<string>',
'table' => '<string>',
'schedule' => '<string>',
'merge' => '<string>',
'column_transformations' => [
[
'column' => '<string>',
'stream' => '<string>',
'new_name' => '<string>',
'encryption_key_name' => '<string>'
]
],
'file_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'file_path' => '<string>',
'file_format' => '<string>',
'file_read_options' => [
],
'jdbc_url' => '<string>',
'jdbc_username' => '<string>',
'jdbc_password' => '<string>',
'jdbc_schema' => '<string>',
'jdbc_table' => '<string>',
'jdbc_query' => '<string>',
'lakehouse_sourceschema' => '<string>',
'lakehouse_sourcetable' => '<string>',
'lakehouse_conditions' => '<string>',
'domain' => '<string>',
'tags' => [
'<string>'
],
'owner_id' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/dataingest/submit"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/dataingest/submit")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/dataingest/submit")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"schema_name\": \"<string>\",\n \"table\": \"<string>\",\n \"schedule\": \"<string>\",\n \"merge\": \"<string>\",\n \"column_transformations\": [\n {\n \"column\": \"<string>\",\n \"stream\": \"<string>\",\n \"new_name\": \"<string>\",\n \"encryption_key_name\": \"<string>\"\n }\n ],\n \"file_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"file_path\": \"<string>\",\n \"file_format\": \"<string>\",\n \"file_read_options\": {},\n \"jdbc_url\": \"<string>\",\n \"jdbc_username\": \"<string>\",\n \"jdbc_password\": \"<string>\",\n \"jdbc_schema\": \"<string>\",\n \"jdbc_table\": \"<string>\",\n \"jdbc_query\": \"<string>\",\n \"lakehouse_sourceschema\": \"<string>\",\n \"lakehouse_sourcetable\": \"<string>\",\n \"lakehouse_conditions\": \"<string>\",\n \"domain\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"owner_id\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"flow_url": "<string>",
"job_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"error": "<string>"
}{
"error": "<string>",
"code": 500
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
OAuth2AuthorizationCodeBearerAPIKeyHeader
The access token received from the authorization server in the OAuth 2.0 flow.
Body
application/json
Request model for data ingestion jobs.
The name of the ingestion job.
Defines the type of data ingestion. Valid values: file, jdbc, or lakehouse.
Available options:
file, jdbc, lakehouse The target schema that receives ingested data.
The target table name for the data.
Defines how the job writes query output to a target table. Valid values: append, overwrite, or merge.
Available options:
append, overwrite, merge The cron expression that specifies when or how often the ingestion runs.
Columns used to match and merge data when the mode is merge.
Optional list of column transformations to apply during ingestion. Supports casting to different data types, renaming columns, and encrypting column values.
Show child attributes
Show child attributes
Unique ID of the uploaded file.
S3 URL to the file you want to ingest.
The format of the file. Valid values: csv, orc, xml, parquet, or xls.
Options for reading the file.
Show child attributes
Show child attributes
Connection URL for the JDBC data source when the format is jdbc.
Username for the JDBC connection.
Password for the JDBC connection.
Type of JDBC ingestion. Valid values: table or query.
Available options:
table, query Source schema in the JDBC database.
Source table in the JDBC database.
SQL query used to extract data via JDBC.
Source schema in the lakehouse when file format is lakehouse.
Source table in the lakehouse.
Filter conditions for the lakehouse source query. For example, salary > 2000, store_id = 1.
The domain to assign to the dataset in DataHub.
Tags to assign to the dataset in DataHub.
The logged in user's email ID.
Was this page helpful?
⌘I