Create and trigger a data migration run
curl --request POST \
--url https://api.example.com/api/migration/data-migration/runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"default_s3_bucket": "<string>",
"tracking_database": "<string>",
"tracking_location": "<string>",
"report_output_location": "<string>",
"s3_access_key": "<string>",
"s3_secret_key": "<string>",
"s3_endpoint": "<string>",
"distcp_mappers": 50,
"distcp_bandwidth": 100,
"email_recipients": "<string>",
"mapr_user": "<string>",
"mapr_ticketfile_location": "<string>",
"kinit_principal": "<string>",
"kinit_keytab": "<string>",
"kinit_password": "<string>",
"inplace_iceberg_migration": false,
"migration_tables": [
{
"database": "<string>",
"table": "*",
"dest_database": "<string>",
"bucket": "<string>",
"endpoint": "<string>",
"partition_filter": "<string>"
}
],
"metadata_tables": [
{
"database": "<string>",
"table": "*",
"inplace_migration": true,
"destination_iceberg_database": "<string>"
}
],
"folder_tables": [
{
"source_path": "<string>",
"target_bucket": "<string>",
"dest_folder": "<string>",
"endpoint": "<string>"
}
]
}
'import requests
url = "https://api.example.com/api/migration/data-migration/runs"
payload = {
"default_s3_bucket": "<string>",
"tracking_database": "<string>",
"tracking_location": "<string>",
"report_output_location": "<string>",
"s3_access_key": "<string>",
"s3_secret_key": "<string>",
"s3_endpoint": "<string>",
"distcp_mappers": 50,
"distcp_bandwidth": 100,
"email_recipients": "<string>",
"mapr_user": "<string>",
"mapr_ticketfile_location": "<string>",
"kinit_principal": "<string>",
"kinit_keytab": "<string>",
"kinit_password": "<string>",
"inplace_iceberg_migration": False,
"migration_tables": [
{
"database": "<string>",
"table": "*",
"dest_database": "<string>",
"bucket": "<string>",
"endpoint": "<string>",
"partition_filter": "<string>"
}
],
"metadata_tables": [
{
"database": "<string>",
"table": "*",
"inplace_migration": True,
"destination_iceberg_database": "<string>"
}
],
"folder_tables": [
{
"source_path": "<string>",
"target_bucket": "<string>",
"dest_folder": "<string>",
"endpoint": "<string>"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
default_s3_bucket: '<string>',
tracking_database: '<string>',
tracking_location: '<string>',
report_output_location: '<string>',
s3_access_key: '<string>',
s3_secret_key: '<string>',
s3_endpoint: '<string>',
distcp_mappers: 50,
distcp_bandwidth: 100,
email_recipients: '<string>',
mapr_user: '<string>',
mapr_ticketfile_location: '<string>',
kinit_principal: '<string>',
kinit_keytab: '<string>',
kinit_password: '<string>',
inplace_iceberg_migration: false,
migration_tables: [
{
database: '<string>',
table: '*',
dest_database: '<string>',
bucket: '<string>',
endpoint: '<string>',
partition_filter: '<string>'
}
],
metadata_tables: [
{
database: '<string>',
table: '*',
inplace_migration: true,
destination_iceberg_database: '<string>'
}
],
folder_tables: [
{
source_path: '<string>',
target_bucket: '<string>',
dest_folder: '<string>',
endpoint: '<string>'
}
]
})
};
fetch('https://api.example.com/api/migration/data-migration/runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/migration/data-migration/runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'default_s3_bucket' => '<string>',
'tracking_database' => '<string>',
'tracking_location' => '<string>',
'report_output_location' => '<string>',
's3_access_key' => '<string>',
's3_secret_key' => '<string>',
's3_endpoint' => '<string>',
'distcp_mappers' => 50,
'distcp_bandwidth' => 100,
'email_recipients' => '<string>',
'mapr_user' => '<string>',
'mapr_ticketfile_location' => '<string>',
'kinit_principal' => '<string>',
'kinit_keytab' => '<string>',
'kinit_password' => '<string>',
'inplace_iceberg_migration' => false,
'migration_tables' => [
[
'database' => '<string>',
'table' => '*',
'dest_database' => '<string>',
'bucket' => '<string>',
'endpoint' => '<string>',
'partition_filter' => '<string>'
]
],
'metadata_tables' => [
[
'database' => '<string>',
'table' => '*',
'inplace_migration' => true,
'destination_iceberg_database' => '<string>'
]
],
'folder_tables' => [
[
'source_path' => '<string>',
'target_bucket' => '<string>',
'dest_folder' => '<string>',
'endpoint' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/migration/data-migration/runs"
payload := strings.NewReader("{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/migration/data-migration/runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/migration/data-migration/runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"run_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"status": "<string>",
"migration_type": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"dag_run_id": "<string>",
"dag_run_ids": [
"<string>"
]
}{
"error": "<string>",
"code": 500
}{
"error": "<string>",
"code": 500
}{
"error": "<string>",
"code": 500
}Data and metadata migration
Create and trigger a data migration run
POST
/
api
/
migration
/
data-migration
/
runs
Create and trigger a data migration run
curl --request POST \
--url https://api.example.com/api/migration/data-migration/runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"default_s3_bucket": "<string>",
"tracking_database": "<string>",
"tracking_location": "<string>",
"report_output_location": "<string>",
"s3_access_key": "<string>",
"s3_secret_key": "<string>",
"s3_endpoint": "<string>",
"distcp_mappers": 50,
"distcp_bandwidth": 100,
"email_recipients": "<string>",
"mapr_user": "<string>",
"mapr_ticketfile_location": "<string>",
"kinit_principal": "<string>",
"kinit_keytab": "<string>",
"kinit_password": "<string>",
"inplace_iceberg_migration": false,
"migration_tables": [
{
"database": "<string>",
"table": "*",
"dest_database": "<string>",
"bucket": "<string>",
"endpoint": "<string>",
"partition_filter": "<string>"
}
],
"metadata_tables": [
{
"database": "<string>",
"table": "*",
"inplace_migration": true,
"destination_iceberg_database": "<string>"
}
],
"folder_tables": [
{
"source_path": "<string>",
"target_bucket": "<string>",
"dest_folder": "<string>",
"endpoint": "<string>"
}
]
}
'import requests
url = "https://api.example.com/api/migration/data-migration/runs"
payload = {
"default_s3_bucket": "<string>",
"tracking_database": "<string>",
"tracking_location": "<string>",
"report_output_location": "<string>",
"s3_access_key": "<string>",
"s3_secret_key": "<string>",
"s3_endpoint": "<string>",
"distcp_mappers": 50,
"distcp_bandwidth": 100,
"email_recipients": "<string>",
"mapr_user": "<string>",
"mapr_ticketfile_location": "<string>",
"kinit_principal": "<string>",
"kinit_keytab": "<string>",
"kinit_password": "<string>",
"inplace_iceberg_migration": False,
"migration_tables": [
{
"database": "<string>",
"table": "*",
"dest_database": "<string>",
"bucket": "<string>",
"endpoint": "<string>",
"partition_filter": "<string>"
}
],
"metadata_tables": [
{
"database": "<string>",
"table": "*",
"inplace_migration": True,
"destination_iceberg_database": "<string>"
}
],
"folder_tables": [
{
"source_path": "<string>",
"target_bucket": "<string>",
"dest_folder": "<string>",
"endpoint": "<string>"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
default_s3_bucket: '<string>',
tracking_database: '<string>',
tracking_location: '<string>',
report_output_location: '<string>',
s3_access_key: '<string>',
s3_secret_key: '<string>',
s3_endpoint: '<string>',
distcp_mappers: 50,
distcp_bandwidth: 100,
email_recipients: '<string>',
mapr_user: '<string>',
mapr_ticketfile_location: '<string>',
kinit_principal: '<string>',
kinit_keytab: '<string>',
kinit_password: '<string>',
inplace_iceberg_migration: false,
migration_tables: [
{
database: '<string>',
table: '*',
dest_database: '<string>',
bucket: '<string>',
endpoint: '<string>',
partition_filter: '<string>'
}
],
metadata_tables: [
{
database: '<string>',
table: '*',
inplace_migration: true,
destination_iceberg_database: '<string>'
}
],
folder_tables: [
{
source_path: '<string>',
target_bucket: '<string>',
dest_folder: '<string>',
endpoint: '<string>'
}
]
})
};
fetch('https://api.example.com/api/migration/data-migration/runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/api/migration/data-migration/runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'default_s3_bucket' => '<string>',
'tracking_database' => '<string>',
'tracking_location' => '<string>',
'report_output_location' => '<string>',
's3_access_key' => '<string>',
's3_secret_key' => '<string>',
's3_endpoint' => '<string>',
'distcp_mappers' => 50,
'distcp_bandwidth' => 100,
'email_recipients' => '<string>',
'mapr_user' => '<string>',
'mapr_ticketfile_location' => '<string>',
'kinit_principal' => '<string>',
'kinit_keytab' => '<string>',
'kinit_password' => '<string>',
'inplace_iceberg_migration' => false,
'migration_tables' => [
[
'database' => '<string>',
'table' => '*',
'dest_database' => '<string>',
'bucket' => '<string>',
'endpoint' => '<string>',
'partition_filter' => '<string>'
]
],
'metadata_tables' => [
[
'database' => '<string>',
'table' => '*',
'inplace_migration' => true,
'destination_iceberg_database' => '<string>'
]
],
'folder_tables' => [
[
'source_path' => '<string>',
'target_bucket' => '<string>',
'dest_folder' => '<string>',
'endpoint' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/api/migration/data-migration/runs"
payload := strings.NewReader("{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/api/migration/data-migration/runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/api/migration/data-migration/runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"default_s3_bucket\": \"<string>\",\n \"tracking_database\": \"<string>\",\n \"tracking_location\": \"<string>\",\n \"report_output_location\": \"<string>\",\n \"s3_access_key\": \"<string>\",\n \"s3_secret_key\": \"<string>\",\n \"s3_endpoint\": \"<string>\",\n \"distcp_mappers\": 50,\n \"distcp_bandwidth\": 100,\n \"email_recipients\": \"<string>\",\n \"mapr_user\": \"<string>\",\n \"mapr_ticketfile_location\": \"<string>\",\n \"kinit_principal\": \"<string>\",\n \"kinit_keytab\": \"<string>\",\n \"kinit_password\": \"<string>\",\n \"inplace_iceberg_migration\": false,\n \"migration_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"dest_database\": \"<string>\",\n \"bucket\": \"<string>\",\n \"endpoint\": \"<string>\",\n \"partition_filter\": \"<string>\"\n }\n ],\n \"metadata_tables\": [\n {\n \"database\": \"<string>\",\n \"table\": \"*\",\n \"inplace_migration\": true,\n \"destination_iceberg_database\": \"<string>\"\n }\n ],\n \"folder_tables\": [\n {\n \"source_path\": \"<string>\",\n \"target_bucket\": \"<string>\",\n \"dest_folder\": \"<string>\",\n \"endpoint\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"run_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"status": "<string>",
"migration_type": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"dag_run_id": "<string>",
"dag_run_ids": [
"<string>"
]
}{
"error": "<string>",
"code": 500
}{
"error": "<string>",
"code": 500
}{
"error": "<string>",
"code": 500
}Authorizations
OAuth2AuthorizationCodeBearerAPIKeyHeader
The access token received from the authorization server in the OAuth 2.0 flow.
Body
application/json
Request to create a data migration run.
Available options:
mapr, hdfs, s3 Available options:
file_only, data_hive_to_hive, data_hive_to_iceberg, metadata_hive_to_iceberg Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?
⌘I

