Create ClickPipe
Create a new ClickPipe.
POST
/
v1
/
organizations
/
{organizationId}
/
services
/
{serviceId}
/
clickpipes
Create ClickPipe
curl --request POST \
--url https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes \
--header 'Authorization: Basic <encoded-value>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"source": {
"kafka": {
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"authentication": "PLAIN",
"credentials": {
"username": "<string>",
"password": "<string>"
},
"type": "confluent",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": [
"<string>"
],
"exactlyOnce": true,
"tombstoneMode": "soft_delete",
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": true,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": true,
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer",
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"serviceAccountKey": "<string>"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": true,
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
},
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"serviceAccountKey": {
"serviceAccountFile": "<string>"
},
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": true,
"ackDeadline": 305
},
"postgres": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": false,
"deleteOnMerge": false
},
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": true,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": [
"<string>"
],
"useCustomSortingKey": true,
"sortingKeys": [
"<string>"
],
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"credentials": {
"serviceAccountFile": "<string>"
},
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"validateSamples": true
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": true,
"tableDefinition": {
"engine": {
"versionColumnId": "<string>",
"columnIds": [
"<string>"
]
},
"sortingKey": [
"<string>"
],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
],
"roles": [
"<string>"
]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": false,
"object_storage_use_cluster_function": true,
"clickhouse_parallel_view_processing": false
},
"startPaused": true
}
'import requests
url = "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"
payload = {
"name": "<string>",
"source": {
"kafka": {
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": { "timestamp": "2021-01-01T00:00" },
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"authentication": "PLAIN",
"credentials": {
"username": "<string>",
"password": "<string>"
},
"type": "confluent",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": ["<string>"],
"exactlyOnce": True,
"tombstoneMode": "soft_delete",
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": True,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": True,
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer",
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"serviceAccountKey": "<string>"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": True,
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
},
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"serviceAccountKey": { "serviceAccountFile": "<string>" },
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": True,
"ackDeadline": 305
},
"postgres": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": False,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": False,
"deleteOnMerge": False
},
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----
...",
"disableTls": False,
"skipCertVerification": False,
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": ["internal_id", "temp_data"],
"useCustomSortingKey": False,
"sortingKeys": ["created_at_date", "event_id"],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": True,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": ["<string>"],
"useCustomSortingKey": True,
"sortingKeys": ["<string>"],
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"credentials": { "serviceAccountFile": "<string>" },
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"validateSamples": True
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": True,
"tableDefinition": {
"engine": {
"versionColumnId": "<string>",
"columnIds": ["<string>"]
},
"sortingKey": ["<string>"],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
],
"roles": ["<string>"]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": False,
"object_storage_use_cluster_function": True,
"clickhouse_parallel_view_processing": False
},
"startPaused": True
}
headers = {
"Authorization": "Basic <encoded-value>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Basic <encoded-value>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
source: {
kafka: {
brokers: '<string>',
topics: 'topic1,topic2',
consumerGroup: 'my-clickpipe-consumer-group',
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
offset: {timestamp: '2021-01-01T00:00'},
schemaRegistry: {
url: 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
authentication: 'PLAIN',
credentials: {username: '<string>', password: '<string>'},
type: 'confluent',
caCertificate: '<string>'
},
caCertificate: '<string>',
reversePrivateEndpointIds: ['<string>'],
exactlyOnce: true,
tombstoneMode: 'soft_delete',
credentials: {username: 'postgres_user', password: 'your_secure_password'},
protobufSchema: 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
},
objectStorage: {
url: 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
delimiter: ',',
compression: 'auto',
isContinuous: true,
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
skipInitialLoad: true,
startAfter: 'events/2026-06-01/',
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
connectionString: 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
path: 'data/logs/*.json',
azureContainerName: 'mycontainer',
accessKey: {accessKeyId: '<string>', secretKey: '<string>'},
serviceAccountKey: '<string>'
},
kinesis: {
streamName: 'my-stream',
region: 'us-east-1',
useEnhancedFanOut: true,
timestamp: 1615766400,
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
schemaRegistry: {
type: 'glue',
glueRegion: 'us-east-1',
glueRegistryName: 'my-registry',
glueRoleArn: 'arn:aws:iam::123456789012:role/MyGlueRegistryRole'
},
accessKey: {accessKeyId: '<string>', secretKey: '<string>'},
protobufSchema: 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
},
pubsub: {
format: 'JSONEachRow',
projectId: 'my-gcp-project',
topic: 'my-topic',
authentication: 'SERVICE_ACCOUNT',
seekType: 'earliest',
serviceAccountKey: {serviceAccountFile: '<string>'},
seekTimestamp: '2026-04-10T12:00:00Z',
filter: '<string>',
enableOrdering: true,
ackDeadline: 305
},
postgres: {
credentials: {username: 'postgres_user', password: 'your_secure_password'},
host: 'my-postgres-server.example.com',
port: 5432,
database: 'production_db',
settings: {
syncIntervalSeconds: 60,
pullBatchSize: 1000,
publicationName: 'clickpipes_publication',
replicationMode: 'cdc',
replicationSlotName: 'clickpipes_slot',
allowNullableColumns: false,
initialLoadParallelism: 1,
snapshotNumRowsPerPartition: 100000,
snapshotNumberOfParallelTables: 1,
enableFailoverSlots: false,
deleteOnMerge: false
},
authentication: 'IAM_ROLE',
iamRole: 'arn:aws:iam::123456789012:role/MyApplicationRole',
tlsHost: 'my-postgres-server.example.com',
caCertificate: '-----BEGIN CERTIFICATE-----\n...',
disableTls: false,
skipCertVerification: false,
tableMappings: [
{
sourceSchemaName: 'public',
sourceTable: 'users',
targetTable: 'public_users',
excludedColumns: ['internal_id', 'temp_data'],
useCustomSortingKey: false,
sortingKeys: ['created_at_date', 'event_id'],
tableEngine: 'ReplacingMergeTree',
partitionKey: 'id',
partitionByExpr: 'toYYYYMM(created_at)'
}
]
},
bigquery: {
snapshotStagingPath: '<string>',
settings: {
replicationMode: 'cdc',
replicationMethod: 'query_based',
syncIntervalSeconds: 60,
queryCdcPullSyncParallelism: 4,
syncDelaySeconds: 60,
pullWindowSizeSeconds: 86400,
syncCron: '0 * * * *',
allowNullableColumns: true,
initialLoadParallelism: 123,
snapshotNumRowsPerPartition: 123,
snapshotNumberOfParallelTables: 123
},
tableMappings: [
{
sourceDatasetName: '<string>',
sourceTable: '<string>',
targetTable: '<string>',
excludedColumns: ['<string>'],
useCustomSortingKey: true,
sortingKeys: ['<string>'],
queryCdcWatermarkColumn: '<string>',
eventsFunction: 'appends'
}
],
credentials: {serviceAccountFile: '<string>'},
authentication: 'SERVICE_ACCOUNT',
projectId: 'my-gcp-project'
},
validateSamples: true
},
destination: {
database: '<string>',
table: '<string>',
managedTable: true,
tableDefinition: {
engine: {versionColumnId: '<string>', columnIds: ['<string>']},
sortingKey: ['<string>'],
partitionBy: '<string>',
primaryKey: '<string>',
ttl: 'toDateTime(event_time) + INTERVAL 30 DAY'
},
columns: [{name: '<string>', type: '<string>'}],
roles: ['<string>']
},
fieldMappings: [{sourceField: '<string>', destinationField: '<string>'}],
scaling: {
replicas: 20,
concurrency: 123,
replicaCpuMillicores: 1062,
replicaMemoryGb: 4.25
},
settings: {
streaming_max_insert_wait_ms: 5000,
object_storage_concurrency: 1,
object_storage_polling_interval_ms: 30000,
object_storage_max_insert_bytes: 10737418240,
object_storage_max_file_count: 100,
clickhouse_max_threads: 8,
clickhouse_max_insert_threads: 1,
clickhouse_min_insert_block_size_bytes: 1073741824,
clickhouse_max_download_threads: 4,
clickhouse_parallel_distributed_insert_select: 2,
kafka_read_committed: false,
object_storage_use_cluster_function: true,
clickhouse_parallel_view_processing: false
},
startPaused: true
})
};
fetch('https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'source' => [
'kafka' => [
'brokers' => '<string>',
'topics' => 'topic1,topic2',
'consumerGroup' => 'my-clickpipe-consumer-group',
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'offset' => [
'timestamp' => '2021-01-01T00:00'
],
'schemaRegistry' => [
'url' => 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
'authentication' => 'PLAIN',
'credentials' => [
'username' => '<string>',
'password' => '<string>'
],
'type' => 'confluent',
'caCertificate' => '<string>'
],
'caCertificate' => '<string>',
'reversePrivateEndpointIds' => [
'<string>'
],
'exactlyOnce' => true,
'tombstoneMode' => 'soft_delete',
'credentials' => [
'username' => 'postgres_user',
'password' => 'your_secure_password'
],
'protobufSchema' => 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
],
'objectStorage' => [
'url' => 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
'delimiter' => ',',
'compression' => 'auto',
'isContinuous' => true,
'queueUrl' => 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
'skipInitialLoad' => true,
'startAfter' => 'events/2026-06-01/',
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'connectionString' => 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
'path' => 'data/logs/*.json',
'azureContainerName' => 'mycontainer',
'accessKey' => [
'accessKeyId' => '<string>',
'secretKey' => '<string>'
],
'serviceAccountKey' => '<string>'
],
'kinesis' => [
'streamName' => 'my-stream',
'region' => 'us-east-1',
'useEnhancedFanOut' => true,
'timestamp' => 1615766400,
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'schemaRegistry' => [
'type' => 'glue',
'glueRegion' => 'us-east-1',
'glueRegistryName' => 'my-registry',
'glueRoleArn' => 'arn:aws:iam::123456789012:role/MyGlueRegistryRole'
],
'accessKey' => [
'accessKeyId' => '<string>',
'secretKey' => '<string>'
],
'protobufSchema' => 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
],
'pubsub' => [
'format' => 'JSONEachRow',
'projectId' => 'my-gcp-project',
'topic' => 'my-topic',
'authentication' => 'SERVICE_ACCOUNT',
'seekType' => 'earliest',
'serviceAccountKey' => [
'serviceAccountFile' => '<string>'
],
'seekTimestamp' => '2026-04-10T12:00:00Z',
'filter' => '<string>',
'enableOrdering' => true,
'ackDeadline' => 305
],
'postgres' => [
'credentials' => [
'username' => 'postgres_user',
'password' => 'your_secure_password'
],
'host' => 'my-postgres-server.example.com',
'port' => 5432,
'database' => 'production_db',
'settings' => [
'syncIntervalSeconds' => 60,
'pullBatchSize' => 1000,
'publicationName' => 'clickpipes_publication',
'replicationMode' => 'cdc',
'replicationSlotName' => 'clickpipes_slot',
'allowNullableColumns' => false,
'initialLoadParallelism' => 1,
'snapshotNumRowsPerPartition' => 100000,
'snapshotNumberOfParallelTables' => 1,
'enableFailoverSlots' => false,
'deleteOnMerge' => false
],
'authentication' => 'IAM_ROLE',
'iamRole' => 'arn:aws:iam::123456789012:role/MyApplicationRole',
'tlsHost' => 'my-postgres-server.example.com',
'caCertificate' => '-----BEGIN CERTIFICATE-----
...',
'disableTls' => false,
'skipCertVerification' => false,
'tableMappings' => [
[
'sourceSchemaName' => 'public',
'sourceTable' => 'users',
'targetTable' => 'public_users',
'excludedColumns' => [
'internal_id',
'temp_data'
],
'useCustomSortingKey' => false,
'sortingKeys' => [
'created_at_date',
'event_id'
],
'tableEngine' => 'ReplacingMergeTree',
'partitionKey' => 'id',
'partitionByExpr' => 'toYYYYMM(created_at)'
]
]
],
'bigquery' => [
'snapshotStagingPath' => '<string>',
'settings' => [
'replicationMode' => 'cdc',
'replicationMethod' => 'query_based',
'syncIntervalSeconds' => 60,
'queryCdcPullSyncParallelism' => 4,
'syncDelaySeconds' => 60,
'pullWindowSizeSeconds' => 86400,
'syncCron' => '0 * * * *',
'allowNullableColumns' => true,
'initialLoadParallelism' => 123,
'snapshotNumRowsPerPartition' => 123,
'snapshotNumberOfParallelTables' => 123
],
'tableMappings' => [
[
'sourceDatasetName' => '<string>',
'sourceTable' => '<string>',
'targetTable' => '<string>',
'excludedColumns' => [
'<string>'
],
'useCustomSortingKey' => true,
'sortingKeys' => [
'<string>'
],
'queryCdcWatermarkColumn' => '<string>',
'eventsFunction' => 'appends'
]
],
'credentials' => [
'serviceAccountFile' => '<string>'
],
'authentication' => 'SERVICE_ACCOUNT',
'projectId' => 'my-gcp-project'
],
'validateSamples' => true
],
'destination' => [
'database' => '<string>',
'table' => '<string>',
'managedTable' => true,
'tableDefinition' => [
'engine' => [
'versionColumnId' => '<string>',
'columnIds' => [
'<string>'
]
],
'sortingKey' => [
'<string>'
],
'partitionBy' => '<string>',
'primaryKey' => '<string>',
'ttl' => 'toDateTime(event_time) + INTERVAL 30 DAY'
],
'columns' => [
[
'name' => '<string>',
'type' => '<string>'
]
],
'roles' => [
'<string>'
]
],
'fieldMappings' => [
[
'sourceField' => '<string>',
'destinationField' => '<string>'
]
],
'scaling' => [
'replicas' => 20,
'concurrency' => 123,
'replicaCpuMillicores' => 1062,
'replicaMemoryGb' => 4.25
],
'settings' => [
'streaming_max_insert_wait_ms' => 5000,
'object_storage_concurrency' => 1,
'object_storage_polling_interval_ms' => 30000,
'object_storage_max_insert_bytes' => 10737418240,
'object_storage_max_file_count' => 100,
'clickhouse_max_threads' => 8,
'clickhouse_max_insert_threads' => 1,
'clickhouse_min_insert_block_size_bytes' => 1073741824,
'clickhouse_max_download_threads' => 4,
'clickhouse_parallel_distributed_insert_select' => 2,
'kafka_read_committed' => false,
'object_storage_use_cluster_function' => true,
'clickhouse_parallel_view_processing' => false
],
'startPaused' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Basic <encoded-value>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Basic <encoded-value>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")
.header("Authorization", "Basic <encoded-value>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Basic <encoded-value>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}"
response = http.request(request)
puts response.read_body{
"status": 200,
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"result": {
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"serviceId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"name": "my_postgres_pipe",
"state": "Running",
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"source": {
"kafka": {
"type": "kafka",
"format": "JSONEachRow",
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"authentication": "PLAIN",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"strategy": "from_beginning",
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"type": "confluent",
"authentication": "PLAIN",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": [
"<string>"
],
"exactlyOnce": true,
"tombstoneMode": "soft_delete"
},
"objectStorage": {
"type": "s3",
"format": "JSONEachRow",
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": true,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": true,
"startAfter": "events/2026-06-01/",
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer"
},
"kinesis": {
"format": "JSONEachRow",
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": true,
"iteratorType": "TRIM_HORIZON",
"timestamp": 1615766400,
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
}
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": true,
"ackDeadline": 305
},
"postgres": {
"type": "postgres",
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": false,
"deleteOnMerge": false
},
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"mysql": {
"host": "my-mysql-server.example.com",
"port": 3306,
"settings": {
"replicationMode": "cdc",
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"replicationMechanism": "GTID",
"useCompression": false,
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"deleteOnMerge": false
},
"tableMappings": [
{
"sourceSchemaName": "my_database",
"sourceTable": "users",
"targetTable": "my_database_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
],
"type": "mysql",
"authentication": "basic",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-mysql-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"serverId": 4242
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": true,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": [
"<string>"
],
"useCustomSortingKey": true,
"sortingKeys": [
"<string>"
],
"tableEngine": "MergeTree",
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"mongodb": {
"uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
"readPreference": "secondaryPreferred",
"tlsHost": "cluster0.example.mongodb.net",
"disableTls": false,
"skipCertVerification": false,
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"settings": {
"replicationMode": "cdc",
"syncIntervalSeconds": 60,
"pullBatchSize": 100000,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"deleteOnMerge": false,
"useJsonNativeFormat": true
},
"tableMappings": [
{
"sourceDatabaseName": "mydb",
"sourceCollection": "users",
"targetTable": "mydb_users",
"tableEngine": "ReplacingMergeTree",
"structuredIngestion": {
"enabled": true,
"dropUnexpectedValues": false,
"columns": [
{
"name": "amount",
"type": "Float64"
}
],
"excludedColumns": [
"internal_notes"
],
"sortingKeys": [
"_id",
"created_at"
],
"partitionByExpr": "toYYYYMM(created_at)"
},
"mongoConfig": {
"persistDeletedDocuments": false
}
}
]
}
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": true,
"tableDefinition": {
"engine": {
"type": "MergeTree",
"versionColumnId": "<string>",
"columnIds": [
"<string>"
]
},
"sortingKey": [
"<string>"
],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": false,
"object_storage_use_cluster_function": true,
"clickhouse_parallel_view_processing": false
},
"createdAt": "2023-11-07T05:31:56Z",
"updatedAt": "2023-11-07T05:31:56Z"
}
}{
"status": 400,
"error": "<string>",
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}{
"status": 500,
"error": "<string>",
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}Create a new ClickPipe.
Permission
The API key must have the
control-plane:service:manage-clickpipes permission.Авторизации
Use key ID and key secret obtained in ClickHouse Cloud console: https://clickhouse.com/docs/cloud/manage/openapi
Параметры пути
ID of the organization that owns the service.
ID of the service to create the ClickPipe for.
Тело
application/json
Name of the ClickPipe.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Field mappings of the ClickPipe. Note that all destination columns must be included in the mappings.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Create the ClickPipe in the Stopped state instead of starting ingestion immediately. Start it later with the state endpoint. Not supported for database ClickPipes.
Была ли эта страница полезной?
⌘I
Create ClickPipe
curl --request POST \
--url https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes \
--header 'Authorization: Basic <encoded-value>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"source": {
"kafka": {
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"authentication": "PLAIN",
"credentials": {
"username": "<string>",
"password": "<string>"
},
"type": "confluent",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": [
"<string>"
],
"exactlyOnce": true,
"tombstoneMode": "soft_delete",
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": true,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": true,
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer",
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"serviceAccountKey": "<string>"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": true,
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
},
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"serviceAccountKey": {
"serviceAccountFile": "<string>"
},
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": true,
"ackDeadline": 305
},
"postgres": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": false,
"deleteOnMerge": false
},
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": true,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": [
"<string>"
],
"useCustomSortingKey": true,
"sortingKeys": [
"<string>"
],
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"credentials": {
"serviceAccountFile": "<string>"
},
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"validateSamples": true
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": true,
"tableDefinition": {
"engine": {
"versionColumnId": "<string>",
"columnIds": [
"<string>"
]
},
"sortingKey": [
"<string>"
],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
],
"roles": [
"<string>"
]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": false,
"object_storage_use_cluster_function": true,
"clickhouse_parallel_view_processing": false
},
"startPaused": true
}
'import requests
url = "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"
payload = {
"name": "<string>",
"source": {
"kafka": {
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": { "timestamp": "2021-01-01T00:00" },
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"authentication": "PLAIN",
"credentials": {
"username": "<string>",
"password": "<string>"
},
"type": "confluent",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": ["<string>"],
"exactlyOnce": True,
"tombstoneMode": "soft_delete",
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": True,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": True,
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer",
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"serviceAccountKey": "<string>"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": True,
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
},
"accessKey": {
"accessKeyId": "<string>",
"secretKey": "<string>"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"serviceAccountKey": { "serviceAccountFile": "<string>" },
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": True,
"ackDeadline": 305
},
"postgres": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": False,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": False,
"deleteOnMerge": False
},
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----
...",
"disableTls": False,
"skipCertVerification": False,
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": ["internal_id", "temp_data"],
"useCustomSortingKey": False,
"sortingKeys": ["created_at_date", "event_id"],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": True,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": ["<string>"],
"useCustomSortingKey": True,
"sortingKeys": ["<string>"],
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"credentials": { "serviceAccountFile": "<string>" },
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"validateSamples": True
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": True,
"tableDefinition": {
"engine": {
"versionColumnId": "<string>",
"columnIds": ["<string>"]
},
"sortingKey": ["<string>"],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
],
"roles": ["<string>"]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": False,
"object_storage_use_cluster_function": True,
"clickhouse_parallel_view_processing": False
},
"startPaused": True
}
headers = {
"Authorization": "Basic <encoded-value>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Basic <encoded-value>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
source: {
kafka: {
brokers: '<string>',
topics: 'topic1,topic2',
consumerGroup: 'my-clickpipe-consumer-group',
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
offset: {timestamp: '2021-01-01T00:00'},
schemaRegistry: {
url: 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
authentication: 'PLAIN',
credentials: {username: '<string>', password: '<string>'},
type: 'confluent',
caCertificate: '<string>'
},
caCertificate: '<string>',
reversePrivateEndpointIds: ['<string>'],
exactlyOnce: true,
tombstoneMode: 'soft_delete',
credentials: {username: 'postgres_user', password: 'your_secure_password'},
protobufSchema: 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
},
objectStorage: {
url: 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
delimiter: ',',
compression: 'auto',
isContinuous: true,
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
skipInitialLoad: true,
startAfter: 'events/2026-06-01/',
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
connectionString: 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
path: 'data/logs/*.json',
azureContainerName: 'mycontainer',
accessKey: {accessKeyId: '<string>', secretKey: '<string>'},
serviceAccountKey: '<string>'
},
kinesis: {
streamName: 'my-stream',
region: 'us-east-1',
useEnhancedFanOut: true,
timestamp: 1615766400,
iamRole: 'arn:aws:iam::123456789012:role/MyRole',
schemaRegistry: {
type: 'glue',
glueRegion: 'us-east-1',
glueRegistryName: 'my-registry',
glueRoleArn: 'arn:aws:iam::123456789012:role/MyGlueRegistryRole'
},
accessKey: {accessKeyId: '<string>', secretKey: '<string>'},
protobufSchema: 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
},
pubsub: {
format: 'JSONEachRow',
projectId: 'my-gcp-project',
topic: 'my-topic',
authentication: 'SERVICE_ACCOUNT',
seekType: 'earliest',
serviceAccountKey: {serviceAccountFile: '<string>'},
seekTimestamp: '2026-04-10T12:00:00Z',
filter: '<string>',
enableOrdering: true,
ackDeadline: 305
},
postgres: {
credentials: {username: 'postgres_user', password: 'your_secure_password'},
host: 'my-postgres-server.example.com',
port: 5432,
database: 'production_db',
settings: {
syncIntervalSeconds: 60,
pullBatchSize: 1000,
publicationName: 'clickpipes_publication',
replicationMode: 'cdc',
replicationSlotName: 'clickpipes_slot',
allowNullableColumns: false,
initialLoadParallelism: 1,
snapshotNumRowsPerPartition: 100000,
snapshotNumberOfParallelTables: 1,
enableFailoverSlots: false,
deleteOnMerge: false
},
authentication: 'IAM_ROLE',
iamRole: 'arn:aws:iam::123456789012:role/MyApplicationRole',
tlsHost: 'my-postgres-server.example.com',
caCertificate: '-----BEGIN CERTIFICATE-----\n...',
disableTls: false,
skipCertVerification: false,
tableMappings: [
{
sourceSchemaName: 'public',
sourceTable: 'users',
targetTable: 'public_users',
excludedColumns: ['internal_id', 'temp_data'],
useCustomSortingKey: false,
sortingKeys: ['created_at_date', 'event_id'],
tableEngine: 'ReplacingMergeTree',
partitionKey: 'id',
partitionByExpr: 'toYYYYMM(created_at)'
}
]
},
bigquery: {
snapshotStagingPath: '<string>',
settings: {
replicationMode: 'cdc',
replicationMethod: 'query_based',
syncIntervalSeconds: 60,
queryCdcPullSyncParallelism: 4,
syncDelaySeconds: 60,
pullWindowSizeSeconds: 86400,
syncCron: '0 * * * *',
allowNullableColumns: true,
initialLoadParallelism: 123,
snapshotNumRowsPerPartition: 123,
snapshotNumberOfParallelTables: 123
},
tableMappings: [
{
sourceDatasetName: '<string>',
sourceTable: '<string>',
targetTable: '<string>',
excludedColumns: ['<string>'],
useCustomSortingKey: true,
sortingKeys: ['<string>'],
queryCdcWatermarkColumn: '<string>',
eventsFunction: 'appends'
}
],
credentials: {serviceAccountFile: '<string>'},
authentication: 'SERVICE_ACCOUNT',
projectId: 'my-gcp-project'
},
validateSamples: true
},
destination: {
database: '<string>',
table: '<string>',
managedTable: true,
tableDefinition: {
engine: {versionColumnId: '<string>', columnIds: ['<string>']},
sortingKey: ['<string>'],
partitionBy: '<string>',
primaryKey: '<string>',
ttl: 'toDateTime(event_time) + INTERVAL 30 DAY'
},
columns: [{name: '<string>', type: '<string>'}],
roles: ['<string>']
},
fieldMappings: [{sourceField: '<string>', destinationField: '<string>'}],
scaling: {
replicas: 20,
concurrency: 123,
replicaCpuMillicores: 1062,
replicaMemoryGb: 4.25
},
settings: {
streaming_max_insert_wait_ms: 5000,
object_storage_concurrency: 1,
object_storage_polling_interval_ms: 30000,
object_storage_max_insert_bytes: 10737418240,
object_storage_max_file_count: 100,
clickhouse_max_threads: 8,
clickhouse_max_insert_threads: 1,
clickhouse_min_insert_block_size_bytes: 1073741824,
clickhouse_max_download_threads: 4,
clickhouse_parallel_distributed_insert_select: 2,
kafka_read_committed: false,
object_storage_use_cluster_function: true,
clickhouse_parallel_view_processing: false
},
startPaused: true
})
};
fetch('https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'source' => [
'kafka' => [
'brokers' => '<string>',
'topics' => 'topic1,topic2',
'consumerGroup' => 'my-clickpipe-consumer-group',
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'offset' => [
'timestamp' => '2021-01-01T00:00'
],
'schemaRegistry' => [
'url' => 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
'authentication' => 'PLAIN',
'credentials' => [
'username' => '<string>',
'password' => '<string>'
],
'type' => 'confluent',
'caCertificate' => '<string>'
],
'caCertificate' => '<string>',
'reversePrivateEndpointIds' => [
'<string>'
],
'exactlyOnce' => true,
'tombstoneMode' => 'soft_delete',
'credentials' => [
'username' => 'postgres_user',
'password' => 'your_secure_password'
],
'protobufSchema' => 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
],
'objectStorage' => [
'url' => 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
'delimiter' => ',',
'compression' => 'auto',
'isContinuous' => true,
'queueUrl' => 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
'skipInitialLoad' => true,
'startAfter' => 'events/2026-06-01/',
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'connectionString' => 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
'path' => 'data/logs/*.json',
'azureContainerName' => 'mycontainer',
'accessKey' => [
'accessKeyId' => '<string>',
'secretKey' => '<string>'
],
'serviceAccountKey' => '<string>'
],
'kinesis' => [
'streamName' => 'my-stream',
'region' => 'us-east-1',
'useEnhancedFanOut' => true,
'timestamp' => 1615766400,
'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
'schemaRegistry' => [
'type' => 'glue',
'glueRegion' => 'us-east-1',
'glueRegistryName' => 'my-registry',
'glueRoleArn' => 'arn:aws:iam::123456789012:role/MyGlueRegistryRole'
],
'accessKey' => [
'accessKeyId' => '<string>',
'secretKey' => '<string>'
],
'protobufSchema' => 'c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30='
],
'pubsub' => [
'format' => 'JSONEachRow',
'projectId' => 'my-gcp-project',
'topic' => 'my-topic',
'authentication' => 'SERVICE_ACCOUNT',
'seekType' => 'earliest',
'serviceAccountKey' => [
'serviceAccountFile' => '<string>'
],
'seekTimestamp' => '2026-04-10T12:00:00Z',
'filter' => '<string>',
'enableOrdering' => true,
'ackDeadline' => 305
],
'postgres' => [
'credentials' => [
'username' => 'postgres_user',
'password' => 'your_secure_password'
],
'host' => 'my-postgres-server.example.com',
'port' => 5432,
'database' => 'production_db',
'settings' => [
'syncIntervalSeconds' => 60,
'pullBatchSize' => 1000,
'publicationName' => 'clickpipes_publication',
'replicationMode' => 'cdc',
'replicationSlotName' => 'clickpipes_slot',
'allowNullableColumns' => false,
'initialLoadParallelism' => 1,
'snapshotNumRowsPerPartition' => 100000,
'snapshotNumberOfParallelTables' => 1,
'enableFailoverSlots' => false,
'deleteOnMerge' => false
],
'authentication' => 'IAM_ROLE',
'iamRole' => 'arn:aws:iam::123456789012:role/MyApplicationRole',
'tlsHost' => 'my-postgres-server.example.com',
'caCertificate' => '-----BEGIN CERTIFICATE-----
...',
'disableTls' => false,
'skipCertVerification' => false,
'tableMappings' => [
[
'sourceSchemaName' => 'public',
'sourceTable' => 'users',
'targetTable' => 'public_users',
'excludedColumns' => [
'internal_id',
'temp_data'
],
'useCustomSortingKey' => false,
'sortingKeys' => [
'created_at_date',
'event_id'
],
'tableEngine' => 'ReplacingMergeTree',
'partitionKey' => 'id',
'partitionByExpr' => 'toYYYYMM(created_at)'
]
]
],
'bigquery' => [
'snapshotStagingPath' => '<string>',
'settings' => [
'replicationMode' => 'cdc',
'replicationMethod' => 'query_based',
'syncIntervalSeconds' => 60,
'queryCdcPullSyncParallelism' => 4,
'syncDelaySeconds' => 60,
'pullWindowSizeSeconds' => 86400,
'syncCron' => '0 * * * *',
'allowNullableColumns' => true,
'initialLoadParallelism' => 123,
'snapshotNumRowsPerPartition' => 123,
'snapshotNumberOfParallelTables' => 123
],
'tableMappings' => [
[
'sourceDatasetName' => '<string>',
'sourceTable' => '<string>',
'targetTable' => '<string>',
'excludedColumns' => [
'<string>'
],
'useCustomSortingKey' => true,
'sortingKeys' => [
'<string>'
],
'queryCdcWatermarkColumn' => '<string>',
'eventsFunction' => 'appends'
]
],
'credentials' => [
'serviceAccountFile' => '<string>'
],
'authentication' => 'SERVICE_ACCOUNT',
'projectId' => 'my-gcp-project'
],
'validateSamples' => true
],
'destination' => [
'database' => '<string>',
'table' => '<string>',
'managedTable' => true,
'tableDefinition' => [
'engine' => [
'versionColumnId' => '<string>',
'columnIds' => [
'<string>'
]
],
'sortingKey' => [
'<string>'
],
'partitionBy' => '<string>',
'primaryKey' => '<string>',
'ttl' => 'toDateTime(event_time) + INTERVAL 30 DAY'
],
'columns' => [
[
'name' => '<string>',
'type' => '<string>'
]
],
'roles' => [
'<string>'
]
],
'fieldMappings' => [
[
'sourceField' => '<string>',
'destinationField' => '<string>'
]
],
'scaling' => [
'replicas' => 20,
'concurrency' => 123,
'replicaCpuMillicores' => 1062,
'replicaMemoryGb' => 4.25
],
'settings' => [
'streaming_max_insert_wait_ms' => 5000,
'object_storage_concurrency' => 1,
'object_storage_polling_interval_ms' => 30000,
'object_storage_max_insert_bytes' => 10737418240,
'object_storage_max_file_count' => 100,
'clickhouse_max_threads' => 8,
'clickhouse_max_insert_threads' => 1,
'clickhouse_min_insert_block_size_bytes' => 1073741824,
'clickhouse_max_download_threads' => 4,
'clickhouse_parallel_distributed_insert_select' => 2,
'kafka_read_committed' => false,
'object_storage_use_cluster_function' => true,
'clickhouse_parallel_view_processing' => false
],
'startPaused' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Basic <encoded-value>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Basic <encoded-value>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")
.header("Authorization", "Basic <encoded-value>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Basic <encoded-value>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"source\": {\n \"kafka\": {\n \"brokers\": \"<string>\",\n \"topics\": \"topic1,topic2\",\n \"consumerGroup\": \"my-clickpipe-consumer-group\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"offset\": {\n \"timestamp\": \"2021-01-01T00:00\"\n },\n \"schemaRegistry\": {\n \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n \"authentication\": \"PLAIN\",\n \"credentials\": {\n \"username\": \"<string>\",\n \"password\": \"<string>\"\n },\n \"type\": \"confluent\",\n \"caCertificate\": \"<string>\"\n },\n \"caCertificate\": \"<string>\",\n \"reversePrivateEndpointIds\": [\n \"<string>\"\n ],\n \"exactlyOnce\": true,\n \"tombstoneMode\": \"soft_delete\",\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"objectStorage\": {\n \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n \"delimiter\": \",\",\n \"compression\": \"auto\",\n \"isContinuous\": true,\n \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n \"skipInitialLoad\": true,\n \"startAfter\": \"events/2026-06-01/\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n \"path\": \"data/logs/*.json\",\n \"azureContainerName\": \"mycontainer\",\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"serviceAccountKey\": \"<string>\"\n },\n \"kinesis\": {\n \"streamName\": \"my-stream\",\n \"region\": \"us-east-1\",\n \"useEnhancedFanOut\": true,\n \"timestamp\": 1615766400,\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n \"schemaRegistry\": {\n \"type\": \"glue\",\n \"glueRegion\": \"us-east-1\",\n \"glueRegistryName\": \"my-registry\",\n \"glueRoleArn\": \"arn:aws:iam::123456789012:role/MyGlueRegistryRole\"\n },\n \"accessKey\": {\n \"accessKeyId\": \"<string>\",\n \"secretKey\": \"<string>\"\n },\n \"protobufSchema\": \"c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=\"\n },\n \"pubsub\": {\n \"format\": \"JSONEachRow\",\n \"projectId\": \"my-gcp-project\",\n \"topic\": \"my-topic\",\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"seekType\": \"earliest\",\n \"serviceAccountKey\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n \"filter\": \"<string>\",\n \"enableOrdering\": true,\n \"ackDeadline\": 305\n },\n \"postgres\": {\n \"credentials\": {\n \"username\": \"postgres_user\",\n \"password\": \"your_secure_password\"\n },\n \"host\": \"my-postgres-server.example.com\",\n \"port\": 5432,\n \"database\": \"production_db\",\n \"settings\": {\n \"syncIntervalSeconds\": 60,\n \"pullBatchSize\": 1000,\n \"publicationName\": \"clickpipes_publication\",\n \"replicationMode\": \"cdc\",\n \"replicationSlotName\": \"clickpipes_slot\",\n \"allowNullableColumns\": false,\n \"initialLoadParallelism\": 1,\n \"snapshotNumRowsPerPartition\": 100000,\n \"snapshotNumberOfParallelTables\": 1,\n \"enableFailoverSlots\": false,\n \"deleteOnMerge\": false\n },\n \"authentication\": \"IAM_ROLE\",\n \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n \"tlsHost\": \"my-postgres-server.example.com\",\n \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"disableTls\": false,\n \"skipCertVerification\": false,\n \"tableMappings\": [\n {\n \"sourceSchemaName\": \"public\",\n \"sourceTable\": \"users\",\n \"targetTable\": \"public_users\",\n \"excludedColumns\": [\n \"internal_id\",\n \"temp_data\"\n ],\n \"useCustomSortingKey\": false,\n \"sortingKeys\": [\n \"created_at_date\",\n \"event_id\"\n ],\n \"tableEngine\": \"ReplacingMergeTree\",\n \"partitionKey\": \"id\",\n \"partitionByExpr\": \"toYYYYMM(created_at)\"\n }\n ]\n },\n \"bigquery\": {\n \"snapshotStagingPath\": \"<string>\",\n \"settings\": {\n \"replicationMode\": \"cdc\",\n \"replicationMethod\": \"query_based\",\n \"syncIntervalSeconds\": 60,\n \"queryCdcPullSyncParallelism\": 4,\n \"syncDelaySeconds\": 60,\n \"pullWindowSizeSeconds\": 86400,\n \"syncCron\": \"0 * * * *\",\n \"allowNullableColumns\": true,\n \"initialLoadParallelism\": 123,\n \"snapshotNumRowsPerPartition\": 123,\n \"snapshotNumberOfParallelTables\": 123\n },\n \"tableMappings\": [\n {\n \"sourceDatasetName\": \"<string>\",\n \"sourceTable\": \"<string>\",\n \"targetTable\": \"<string>\",\n \"excludedColumns\": [\n \"<string>\"\n ],\n \"useCustomSortingKey\": true,\n \"sortingKeys\": [\n \"<string>\"\n ],\n \"queryCdcWatermarkColumn\": \"<string>\",\n \"eventsFunction\": \"appends\"\n }\n ],\n \"credentials\": {\n \"serviceAccountFile\": \"<string>\"\n },\n \"authentication\": \"SERVICE_ACCOUNT\",\n \"projectId\": \"my-gcp-project\"\n },\n \"validateSamples\": true\n },\n \"destination\": {\n \"database\": \"<string>\",\n \"table\": \"<string>\",\n \"managedTable\": true,\n \"tableDefinition\": {\n \"engine\": {\n \"versionColumnId\": \"<string>\",\n \"columnIds\": [\n \"<string>\"\n ]\n },\n \"sortingKey\": [\n \"<string>\"\n ],\n \"partitionBy\": \"<string>\",\n \"primaryKey\": \"<string>\",\n \"ttl\": \"toDateTime(event_time) + INTERVAL 30 DAY\"\n },\n \"columns\": [\n {\n \"name\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"roles\": [\n \"<string>\"\n ]\n },\n \"fieldMappings\": [\n {\n \"sourceField\": \"<string>\",\n \"destinationField\": \"<string>\"\n }\n ],\n \"scaling\": {\n \"replicas\": 20,\n \"concurrency\": 123,\n \"replicaCpuMillicores\": 1062,\n \"replicaMemoryGb\": 4.25\n },\n \"settings\": {\n \"streaming_max_insert_wait_ms\": 5000,\n \"object_storage_concurrency\": 1,\n \"object_storage_polling_interval_ms\": 30000,\n \"object_storage_max_insert_bytes\": 10737418240,\n \"object_storage_max_file_count\": 100,\n \"clickhouse_max_threads\": 8,\n \"clickhouse_max_insert_threads\": 1,\n \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n \"clickhouse_max_download_threads\": 4,\n \"clickhouse_parallel_distributed_insert_select\": 2,\n \"kafka_read_committed\": false,\n \"object_storage_use_cluster_function\": true,\n \"clickhouse_parallel_view_processing\": false\n },\n \"startPaused\": true\n}"
response = http.request(request)
puts response.read_body{
"status": 200,
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"result": {
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"serviceId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"name": "my_postgres_pipe",
"state": "Running",
"scaling": {
"replicas": 20,
"concurrency": 123,
"replicaCpuMillicores": 1062,
"replicaMemoryGb": 4.25
},
"source": {
"kafka": {
"type": "kafka",
"format": "JSONEachRow",
"brokers": "<string>",
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"authentication": "PLAIN",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"strategy": "from_beginning",
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
"type": "confluent",
"authentication": "PLAIN",
"caCertificate": "<string>"
},
"caCertificate": "<string>",
"reversePrivateEndpointIds": [
"<string>"
],
"exactlyOnce": true,
"tombstoneMode": "soft_delete"
},
"objectStorage": {
"type": "s3",
"format": "JSONEachRow",
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"isContinuous": true,
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"skipInitialLoad": true,
"startAfter": "events/2026-06-01/",
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer"
},
"kinesis": {
"format": "JSONEachRow",
"streamName": "my-stream",
"region": "us-east-1",
"useEnhancedFanOut": true,
"iteratorType": "TRIM_HORIZON",
"timestamp": 1615766400,
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"type": "glue",
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
}
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"seekTimestamp": "2026-04-10T12:00:00Z",
"filter": "<string>",
"enableOrdering": true,
"ackDeadline": 305
},
"postgres": {
"type": "postgres",
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"enableFailoverSlots": false,
"deleteOnMerge": false
},
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"mysql": {
"host": "my-mysql-server.example.com",
"port": 3306,
"settings": {
"replicationMode": "cdc",
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"replicationMechanism": "GTID",
"useCompression": false,
"allowNullableColumns": false,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"deleteOnMerge": false
},
"tableMappings": [
{
"sourceSchemaName": "my_database",
"sourceTable": "users",
"targetTable": "my_database_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"useCustomSortingKey": false,
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
],
"type": "mysql",
"authentication": "basic",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-mysql-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"disableTls": false,
"skipCertVerification": false,
"serverId": 4242
},
"bigquery": {
"snapshotStagingPath": "<string>",
"settings": {
"replicationMode": "cdc",
"replicationMethod": "query_based",
"syncIntervalSeconds": 60,
"queryCdcPullSyncParallelism": 4,
"syncDelaySeconds": 60,
"pullWindowSizeSeconds": 86400,
"syncCron": "0 * * * *",
"allowNullableColumns": true,
"initialLoadParallelism": 123,
"snapshotNumRowsPerPartition": 123,
"snapshotNumberOfParallelTables": 123
},
"tableMappings": [
{
"sourceDatasetName": "<string>",
"sourceTable": "<string>",
"targetTable": "<string>",
"excludedColumns": [
"<string>"
],
"useCustomSortingKey": true,
"sortingKeys": [
"<string>"
],
"tableEngine": "MergeTree",
"queryCdcWatermarkColumn": "<string>",
"eventsFunction": "appends"
}
],
"authentication": "SERVICE_ACCOUNT",
"projectId": "my-gcp-project"
},
"mongodb": {
"uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
"readPreference": "secondaryPreferred",
"tlsHost": "cluster0.example.mongodb.net",
"disableTls": false,
"skipCertVerification": false,
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"settings": {
"replicationMode": "cdc",
"syncIntervalSeconds": 60,
"pullBatchSize": 100000,
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"deleteOnMerge": false,
"useJsonNativeFormat": true
},
"tableMappings": [
{
"sourceDatabaseName": "mydb",
"sourceCollection": "users",
"targetTable": "mydb_users",
"tableEngine": "ReplacingMergeTree",
"structuredIngestion": {
"enabled": true,
"dropUnexpectedValues": false,
"columns": [
{
"name": "amount",
"type": "Float64"
}
],
"excludedColumns": [
"internal_notes"
],
"sortingKeys": [
"_id",
"created_at"
],
"partitionByExpr": "toYYYYMM(created_at)"
},
"mongoConfig": {
"persistDeletedDocuments": false
}
}
]
}
},
"destination": {
"database": "<string>",
"table": "<string>",
"managedTable": true,
"tableDefinition": {
"engine": {
"type": "MergeTree",
"versionColumnId": "<string>",
"columnIds": [
"<string>"
]
},
"sortingKey": [
"<string>"
],
"partitionBy": "<string>",
"primaryKey": "<string>",
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
},
"columns": [
{
"name": "<string>",
"type": "<string>"
}
]
},
"fieldMappings": [
{
"sourceField": "<string>",
"destinationField": "<string>"
}
],
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"kafka_read_committed": false,
"object_storage_use_cluster_function": true,
"clickhouse_parallel_view_processing": false
},
"createdAt": "2023-11-07T05:31:56Z",
"updatedAt": "2023-11-07T05:31:56Z"
}
}{
"status": 400,
"error": "<string>",
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}{
"status": 500,
"error": "<string>",
"requestId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}