Create ClickPipe

curl --request POST \
  --url https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes \
  --header 'Authorization: Basic <encoded-value>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "<string>",
  "source": {
    "kafka": {
      "brokers": "<string>",
      "topics": "<string>",
      "consumerGroup": "my-clickpipe-consumer-group",
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "offset": {
        "timestamp": "2021-01-01T00:00"
      },
      "schemaRegistry": {
        "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
        "authentication": "PLAIN",
        "caCertificate": "<string>",
        "credentials": {
          "username": "<string>",
          "password": "<string>"
        }
      },
      "caCertificate": "<string>",
      "reversePrivateEndpointIds": [
        "<string>"
      ],
      "exactlyOnce": true,
      "credentials": {
        "username": "postgres_user",
        "password": "your_secure_password"
      }
    },
    "objectStorage": {
      "url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
      "delimiter": ",",
      "compression": "auto",
      "isContinuous": true,
      "queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
      "skipInitialLoad": true,
      "startAfter": "events/2026-06-01/",
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
      "path": "data/logs/*.json",
      "azureContainerName": "mycontainer",
      "accessKey": {
        "accessKeyId": "<string>",
        "secretKey": "<string>"
      },
      "serviceAccountKey": "<string>"
    },
    "kinesis": {
      "streamName": "my-stream",
      "region": "us-east-1",
      "useEnhancedFanOut": true,
      "timestamp": 1615766400,
      "iamRole": "arn:aws:iam::123456789012:role/MyRole",
      "accessKey": {
        "accessKeyId": "<string>",
        "secretKey": "<string>"
      }
    },
    "pubsub": {
      "format": "JSONEachRow",
      "projectId": "my-gcp-project",
      "topic": "my-topic",
      "authentication": "SERVICE_ACCOUNT",
      "seekType": "earliest",
      "serviceAccountKey": {
        "serviceAccountFile": "<string>"
      },
      "seekTimestamp": "2026-04-10T12:00:00Z",
      "filter": "<string>",
      "enableOrdering": true,
      "ackDeadline": 305
    },
    "postgres": {
      "credentials": {
        "username": "postgres_user",
        "password": "your_secure_password"
      },
      "host": "my-postgres-server.example.com",
      "port": 5432,
      "database": "production_db",
      "settings": {
        "syncIntervalSeconds": 60,
        "pullBatchSize": 1000,
        "publicationName": "clickpipes_publication",
        "replicationMode": "cdc",
        "replicationSlotName": "clickpipes_slot",
        "allowNullableColumns": false,
        "initialLoadParallelism": 1,
        "snapshotNumRowsPerPartition": 100000,
        "snapshotNumberOfParallelTables": 1,
        "enableFailoverSlots": false,
        "deleteOnMerge": false
      },
      "authentication": "IAM_ROLE",
      "iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
      "tlsHost": "my-postgres-server.example.com",
      "caCertificate": "-----BEGIN CERTIFICATE-----\n...",
      "disableTls": false,
      "skipCertVerification": false,
      "tableMappings": [
        {
          "sourceSchemaName": "public",
          "sourceTable": "users",
          "targetTable": "public_users",
          "excludedColumns": [
            "internal_id",
            "temp_data"
          ],
          "useCustomSortingKey": false,
          "sortingKeys": [
            "created_at_date",
            "event_id"
          ],
          "tableEngine": "ReplacingMergeTree",
          "partitionKey": "id"
        }
      ]
    },
    "mysql": {
      "host": "my-mysql-server.example.com",
      "port": 3306,
      "settings": {
        "replicationMode": "cdc",
        "syncIntervalSeconds": 60,
        "pullBatchSize": 1000,
        "replicationMechanism": "GTID",
        "useCompression": false,
        "allowNullableColumns": false,
        "initialLoadParallelism": 1,
        "snapshotNumRowsPerPartition": 100000,
        "snapshotNumberOfParallelTables": 1,
        "deleteOnMerge": false
      },
      "tableMappings": [
        {
          "sourceSchemaName": "my_database",
          "sourceTable": "users",
          "targetTable": "my_database_users",
          "excludedColumns": [
            "internal_id",
            "temp_data"
          ],
          "useCustomSortingKey": false,
          "sortingKeys": [
            "created_at_date",
            "event_id"
          ],
          "tableEngine": "ReplacingMergeTree",
          "partitionKey": "id"
        }
      ],
      "credentials": {
        "username": "postgres_user",
        "password": "your_secure_password"
      },
      "authentication": "basic",
      "iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
      "tlsHost": "my-mysql-server.example.com",
      "caCertificate": "-----BEGIN CERTIFICATE-----\n...",
      "disableTls": false,
      "skipCertVerification": false,
      "serverId": 4242
    },
    "bigquery": {
      "snapshotStagingPath": "<string>",
      "settings": {
        "replicationMode": "snapshot",
        "allowNullableColumns": true,
        "initialLoadParallelism": 123,
        "snapshotNumRowsPerPartition": 123,
        "snapshotNumberOfParallelTables": 123
      },
      "tableMappings": [
        {
          "sourceDatasetName": "<string>",
          "sourceTable": "<string>",
          "targetTable": "<string>",
          "excludedColumns": [
            "<string>"
          ],
          "useCustomSortingKey": true,
          "sortingKeys": [
            "<string>"
          ]
        }
      ],
      "credentials": {
        "serviceAccountFile": "<string>"
      }
    },
    "mongodb": {
      "uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
      "readPreference": "secondaryPreferred",
      "settings": {
        "replicationMode": "cdc",
        "syncIntervalSeconds": 60,
        "pullBatchSize": 100000,
        "snapshotNumRowsPerPartition": 100000,
        "snapshotNumberOfParallelTables": 1,
        "deleteOnMerge": false,
        "useJsonNativeFormat": true
      },
      "tableMappings": [
        {
          "sourceDatabaseName": "mydb",
          "sourceCollection": "users",
          "targetTable": "mydb_users",
          "tableEngine": "ReplacingMergeTree"
        }
      ],
      "credentials": {
        "username": "postgres_user",
        "password": "your_secure_password"
      },
      "tlsHost": "cluster0.example.mongodb.net",
      "disableTls": false,
      "skipCertVerification": false,
      "caCertificate": "-----BEGIN CERTIFICATE-----\n..."
    },
    "validateSamples": true
  },
  "destination": {
    "database": "<string>",
    "table": "<string>",
    "managedTable": true,
    "tableDefinition": {
      "engine": {
        "versionColumnId": "<string>",
        "columnIds": [
          "<string>"
        ]
      },
      "sortingKey": [
        "<string>"
      ],
      "partitionBy": "<string>",
      "primaryKey": "<string>"
    },
    "columns": [
      {
        "name": "<string>",
        "type": "<string>"
      }
    ],
    "roles": [
      "<string>"
    ]
  },
  "fieldMappings": [
    {
      "sourceField": "<string>",
      "destinationField": "<string>"
    }
  ],
  "scaling": {
    "replicas": 20,
    "concurrency": 123,
    "replicaCpuMillicores": 1062,
    "replicaMemoryGb": 4.25
  },
  "settings": {
    "streaming_max_insert_wait_ms": 5000,
    "object_storage_concurrency": 1,
    "object_storage_polling_interval_ms": 30000,
    "object_storage_max_insert_bytes": 10737418240,
    "object_storage_max_file_count": 100,
    "clickhouse_max_threads": 8,
    "clickhouse_max_insert_threads": 1,
    "clickhouse_min_insert_block_size_bytes": 1073741824,
    "clickhouse_max_download_threads": 4,
    "clickhouse_parallel_distributed_insert_select": 2,
    "object_storage_use_cluster_function": true,
    "clickhouse_parallel_view_processing": false
  }
}
'

import requests

url = "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"

payload = {
    "name": "<string>",
    "source": {
        "kafka": {
            "brokers": "<string>",
            "topics": "<string>",
            "consumerGroup": "my-clickpipe-consumer-group",
            "iamRole": "arn:aws:iam::123456789012:role/MyRole",
            "offset": { "timestamp": "2021-01-01T00:00" },
            "schemaRegistry": {
                "url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004",
                "authentication": "PLAIN",
                "caCertificate": "<string>",
                "credentials": {
                    "username": "<string>",
                    "password": "<string>"
                }
            },
            "caCertificate": "<string>",
            "reversePrivateEndpointIds": ["<string>"],
            "exactlyOnce": True,
            "credentials": {
                "username": "postgres_user",
                "password": "your_secure_password"
            }
        },
        "objectStorage": {
            "url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
            "delimiter": ",",
            "compression": "auto",
            "isContinuous": True,
            "queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
            "skipInitialLoad": True,
            "startAfter": "events/2026-06-01/",
            "iamRole": "arn:aws:iam::123456789012:role/MyRole",
            "connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
            "path": "data/logs/*.json",
            "azureContainerName": "mycontainer",
            "accessKey": {
                "accessKeyId": "<string>",
                "secretKey": "<string>"
            },
            "serviceAccountKey": "<string>"
        },
        "kinesis": {
            "streamName": "my-stream",
            "region": "us-east-1",
            "useEnhancedFanOut": True,
            "timestamp": 1615766400,
            "iamRole": "arn:aws:iam::123456789012:role/MyRole",
            "accessKey": {
                "accessKeyId": "<string>",
                "secretKey": "<string>"
            }
        },
        "pubsub": {
            "format": "JSONEachRow",
            "projectId": "my-gcp-project",
            "topic": "my-topic",
            "authentication": "SERVICE_ACCOUNT",
            "seekType": "earliest",
            "serviceAccountKey": { "serviceAccountFile": "<string>" },
            "seekTimestamp": "2026-04-10T12:00:00Z",
            "filter": "<string>",
            "enableOrdering": True,
            "ackDeadline": 305
        },
        "postgres": {
            "credentials": {
                "username": "postgres_user",
                "password": "your_secure_password"
            },
            "host": "my-postgres-server.example.com",
            "port": 5432,
            "database": "production_db",
            "settings": {
                "syncIntervalSeconds": 60,
                "pullBatchSize": 1000,
                "publicationName": "clickpipes_publication",
                "replicationMode": "cdc",
                "replicationSlotName": "clickpipes_slot",
                "allowNullableColumns": False,
                "initialLoadParallelism": 1,
                "snapshotNumRowsPerPartition": 100000,
                "snapshotNumberOfParallelTables": 1,
                "enableFailoverSlots": False,
                "deleteOnMerge": False
            },
            "authentication": "IAM_ROLE",
            "iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
            "tlsHost": "my-postgres-server.example.com",
            "caCertificate": "-----BEGIN CERTIFICATE-----
...",
            "disableTls": False,
            "skipCertVerification": False,
            "tableMappings": [
                {
                    "sourceSchemaName": "public",
                    "sourceTable": "users",
                    "targetTable": "public_users",
                    "excludedColumns": ["internal_id", "temp_data"],
                    "useCustomSortingKey": False,
                    "sortingKeys": ["created_at_date", "event_id"],
                    "tableEngine": "ReplacingMergeTree",
                    "partitionKey": "id"
                }
            ]
        },
        "mysql": {
            "host": "my-mysql-server.example.com",
            "port": 3306,
            "settings": {
                "replicationMode": "cdc",
                "syncIntervalSeconds": 60,
                "pullBatchSize": 1000,
                "replicationMechanism": "GTID",
                "useCompression": False,
                "allowNullableColumns": False,
                "initialLoadParallelism": 1,
                "snapshotNumRowsPerPartition": 100000,
                "snapshotNumberOfParallelTables": 1,
                "deleteOnMerge": False
            },
            "tableMappings": [
                {
                    "sourceSchemaName": "my_database",
                    "sourceTable": "users",
                    "targetTable": "my_database_users",
                    "excludedColumns": ["internal_id", "temp_data"],
                    "useCustomSortingKey": False,
                    "sortingKeys": ["created_at_date", "event_id"],
                    "tableEngine": "ReplacingMergeTree",
                    "partitionKey": "id"
                }
            ],
            "credentials": {
                "username": "postgres_user",
                "password": "your_secure_password"
            },
            "authentication": "basic",
            "iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
            "tlsHost": "my-mysql-server.example.com",
            "caCertificate": "-----BEGIN CERTIFICATE-----
...",
            "disableTls": False,
            "skipCertVerification": False,
            "serverId": 4242
        },
        "bigquery": {
            "snapshotStagingPath": "<string>",
            "settings": {
                "replicationMode": "snapshot",
                "allowNullableColumns": True,
                "initialLoadParallelism": 123,
                "snapshotNumRowsPerPartition": 123,
                "snapshotNumberOfParallelTables": 123
            },
            "tableMappings": [
                {
                    "sourceDatasetName": "<string>",
                    "sourceTable": "<string>",
                    "targetTable": "<string>",
                    "excludedColumns": ["<string>"],
                    "useCustomSortingKey": True,
                    "sortingKeys": ["<string>"]
                }
            ],
            "credentials": { "serviceAccountFile": "<string>" }
        },
        "mongodb": {
            "uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
            "readPreference": "secondaryPreferred",
            "settings": {
                "replicationMode": "cdc",
                "syncIntervalSeconds": 60,
                "pullBatchSize": 100000,
                "snapshotNumRowsPerPartition": 100000,
                "snapshotNumberOfParallelTables": 1,
                "deleteOnMerge": False,
                "useJsonNativeFormat": True
            },
            "tableMappings": [
                {
                    "sourceDatabaseName": "mydb",
                    "sourceCollection": "users",
                    "targetTable": "mydb_users",
                    "tableEngine": "ReplacingMergeTree"
                }
            ],
            "credentials": {
                "username": "postgres_user",
                "password": "your_secure_password"
            },
            "tlsHost": "cluster0.example.mongodb.net",
            "disableTls": False,
            "skipCertVerification": False,
            "caCertificate": "-----BEGIN CERTIFICATE-----
..."
        },
        "validateSamples": True
    },
    "destination": {
        "database": "<string>",
        "table": "<string>",
        "managedTable": True,
        "tableDefinition": {
            "engine": {
                "versionColumnId": "<string>",
                "columnIds": ["<string>"]
            },
            "sortingKey": ["<string>"],
            "partitionBy": "<string>",
            "primaryKey": "<string>"
        },
        "columns": [
            {
                "name": "<string>",
                "type": "<string>"
            }
        ],
        "roles": ["<string>"]
    },
    "fieldMappings": [
        {
            "sourceField": "<string>",
            "destinationField": "<string>"
        }
    ],
    "scaling": {
        "replicas": 20,
        "concurrency": 123,
        "replicaCpuMillicores": 1062,
        "replicaMemoryGb": 4.25
    },
    "settings": {
        "streaming_max_insert_wait_ms": 5000,
        "object_storage_concurrency": 1,
        "object_storage_polling_interval_ms": 30000,
        "object_storage_max_insert_bytes": 10737418240,
        "object_storage_max_file_count": 100,
        "clickhouse_max_threads": 8,
        "clickhouse_max_insert_threads": 1,
        "clickhouse_min_insert_block_size_bytes": 1073741824,
        "clickhouse_max_download_threads": 4,
        "clickhouse_parallel_distributed_insert_select": 2,
        "object_storage_use_cluster_function": True,
        "clickhouse_parallel_view_processing": False
    }
}
headers = {
    "Authorization": "Basic <encoded-value>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Basic <encoded-value>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: '<string>',
    source: {
      kafka: {
        brokers: '<string>',
        topics: '<string>',
        consumerGroup: 'my-clickpipe-consumer-group',
        iamRole: 'arn:aws:iam::123456789012:role/MyRole',
        offset: {timestamp: '2021-01-01T00:00'},
        schemaRegistry: {
          url: 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
          authentication: 'PLAIN',
          caCertificate: '<string>',
          credentials: {username: '<string>', password: '<string>'}
        },
        caCertificate: '<string>',
        reversePrivateEndpointIds: ['<string>'],
        exactlyOnce: true,
        credentials: {username: 'postgres_user', password: 'your_secure_password'}
      },
      objectStorage: {
        url: 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
        delimiter: ',',
        compression: 'auto',
        isContinuous: true,
        queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
        skipInitialLoad: true,
        startAfter: 'events/2026-06-01/',
        iamRole: 'arn:aws:iam::123456789012:role/MyRole',
        connectionString: 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
        path: 'data/logs/*.json',
        azureContainerName: 'mycontainer',
        accessKey: {accessKeyId: '<string>', secretKey: '<string>'},
        serviceAccountKey: '<string>'
      },
      kinesis: {
        streamName: 'my-stream',
        region: 'us-east-1',
        useEnhancedFanOut: true,
        timestamp: 1615766400,
        iamRole: 'arn:aws:iam::123456789012:role/MyRole',
        accessKey: {accessKeyId: '<string>', secretKey: '<string>'}
      },
      pubsub: {
        format: 'JSONEachRow',
        projectId: 'my-gcp-project',
        topic: 'my-topic',
        authentication: 'SERVICE_ACCOUNT',
        seekType: 'earliest',
        serviceAccountKey: {serviceAccountFile: '<string>'},
        seekTimestamp: '2026-04-10T12:00:00Z',
        filter: '<string>',
        enableOrdering: true,
        ackDeadline: 305
      },
      postgres: {
        credentials: {username: 'postgres_user', password: 'your_secure_password'},
        host: 'my-postgres-server.example.com',
        port: 5432,
        database: 'production_db',
        settings: {
          syncIntervalSeconds: 60,
          pullBatchSize: 1000,
          publicationName: 'clickpipes_publication',
          replicationMode: 'cdc',
          replicationSlotName: 'clickpipes_slot',
          allowNullableColumns: false,
          initialLoadParallelism: 1,
          snapshotNumRowsPerPartition: 100000,
          snapshotNumberOfParallelTables: 1,
          enableFailoverSlots: false,
          deleteOnMerge: false
        },
        authentication: 'IAM_ROLE',
        iamRole: 'arn:aws:iam::123456789012:role/MyApplicationRole',
        tlsHost: 'my-postgres-server.example.com',
        caCertificate: '-----BEGIN CERTIFICATE-----\n...',
        disableTls: false,
        skipCertVerification: false,
        tableMappings: [
          {
            sourceSchemaName: 'public',
            sourceTable: 'users',
            targetTable: 'public_users',
            excludedColumns: ['internal_id', 'temp_data'],
            useCustomSortingKey: false,
            sortingKeys: ['created_at_date', 'event_id'],
            tableEngine: 'ReplacingMergeTree',
            partitionKey: 'id'
          }
        ]
      },
      mysql: {
        host: 'my-mysql-server.example.com',
        port: 3306,
        settings: {
          replicationMode: 'cdc',
          syncIntervalSeconds: 60,
          pullBatchSize: 1000,
          replicationMechanism: 'GTID',
          useCompression: false,
          allowNullableColumns: false,
          initialLoadParallelism: 1,
          snapshotNumRowsPerPartition: 100000,
          snapshotNumberOfParallelTables: 1,
          deleteOnMerge: false
        },
        tableMappings: [
          {
            sourceSchemaName: 'my_database',
            sourceTable: 'users',
            targetTable: 'my_database_users',
            excludedColumns: ['internal_id', 'temp_data'],
            useCustomSortingKey: false,
            sortingKeys: ['created_at_date', 'event_id'],
            tableEngine: 'ReplacingMergeTree',
            partitionKey: 'id'
          }
        ],
        credentials: {username: 'postgres_user', password: 'your_secure_password'},
        authentication: 'basic',
        iamRole: 'arn:aws:iam::123456789012:role/MyApplicationRole',
        tlsHost: 'my-mysql-server.example.com',
        caCertificate: '-----BEGIN CERTIFICATE-----\n...',
        disableTls: false,
        skipCertVerification: false,
        serverId: 4242
      },
      bigquery: {
        snapshotStagingPath: '<string>',
        settings: {
          replicationMode: 'snapshot',
          allowNullableColumns: true,
          initialLoadParallelism: 123,
          snapshotNumRowsPerPartition: 123,
          snapshotNumberOfParallelTables: 123
        },
        tableMappings: [
          {
            sourceDatasetName: '<string>',
            sourceTable: '<string>',
            targetTable: '<string>',
            excludedColumns: ['<string>'],
            useCustomSortingKey: true,
            sortingKeys: ['<string>']
          }
        ],
        credentials: {serviceAccountFile: '<string>'}
      },
      mongodb: {
        uri: 'mongodb+srv://cluster0.example.mongodb.net/mydb',
        readPreference: 'secondaryPreferred',
        settings: {
          replicationMode: 'cdc',
          syncIntervalSeconds: 60,
          pullBatchSize: 100000,
          snapshotNumRowsPerPartition: 100000,
          snapshotNumberOfParallelTables: 1,
          deleteOnMerge: false,
          useJsonNativeFormat: true
        },
        tableMappings: [
          {
            sourceDatabaseName: 'mydb',
            sourceCollection: 'users',
            targetTable: 'mydb_users',
            tableEngine: 'ReplacingMergeTree'
          }
        ],
        credentials: {username: 'postgres_user', password: 'your_secure_password'},
        tlsHost: 'cluster0.example.mongodb.net',
        disableTls: false,
        skipCertVerification: false,
        caCertificate: '-----BEGIN CERTIFICATE-----\n...'
      },
      validateSamples: true
    },
    destination: {
      database: '<string>',
      table: '<string>',
      managedTable: true,
      tableDefinition: {
        engine: {versionColumnId: '<string>', columnIds: ['<string>']},
        sortingKey: ['<string>'],
        partitionBy: '<string>',
        primaryKey: '<string>'
      },
      columns: [{name: '<string>', type: '<string>'}],
      roles: ['<string>']
    },
    fieldMappings: [{sourceField: '<string>', destinationField: '<string>'}],
    scaling: {
      replicas: 20,
      concurrency: 123,
      replicaCpuMillicores: 1062,
      replicaMemoryGb: 4.25
    },
    settings: {
      streaming_max_insert_wait_ms: 5000,
      object_storage_concurrency: 1,
      object_storage_polling_interval_ms: 30000,
      object_storage_max_insert_bytes: 10737418240,
      object_storage_max_file_count: 100,
      clickhouse_max_threads: 8,
      clickhouse_max_insert_threads: 1,
      clickhouse_min_insert_block_size_bytes: 1073741824,
      clickhouse_max_download_threads: 4,
      clickhouse_parallel_distributed_insert_select: 2,
      object_storage_use_cluster_function: true,
      clickhouse_parallel_view_processing: false
    }
  })
};

fetch('https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => '<string>',
    'source' => [
        'kafka' => [
                'brokers' => '<string>',
                'topics' => '<string>',
                'consumerGroup' => 'my-clickpipe-consumer-group',
                'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
                'offset' => [
                                'timestamp' => '2021-01-01T00:00'
                ],
                'schemaRegistry' => [
                                'url' => 'https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004',
                                'authentication' => 'PLAIN',
                                'caCertificate' => '<string>',
                                'credentials' => [
                                                                'username' => '<string>',
                                                                'password' => '<string>'
                                ]
                ],
                'caCertificate' => '<string>',
                'reversePrivateEndpointIds' => [
                                '<string>'
                ],
                'exactlyOnce' => true,
                'credentials' => [
                                'username' => 'postgres_user',
                                'password' => 'your_secure_password'
                ]
        ],
        'objectStorage' => [
                'url' => 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz',
                'delimiter' => ',',
                'compression' => 'auto',
                'isContinuous' => true,
                'queueUrl' => 'https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue',
                'skipInitialLoad' => true,
                'startAfter' => 'events/2026-06-01/',
                'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
                'connectionString' => 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net',
                'path' => 'data/logs/*.json',
                'azureContainerName' => 'mycontainer',
                'accessKey' => [
                                'accessKeyId' => '<string>',
                                'secretKey' => '<string>'
                ],
                'serviceAccountKey' => '<string>'
        ],
        'kinesis' => [
                'streamName' => 'my-stream',
                'region' => 'us-east-1',
                'useEnhancedFanOut' => true,
                'timestamp' => 1615766400,
                'iamRole' => 'arn:aws:iam::123456789012:role/MyRole',
                'accessKey' => [
                                'accessKeyId' => '<string>',
                                'secretKey' => '<string>'
                ]
        ],
        'pubsub' => [
                'format' => 'JSONEachRow',
                'projectId' => 'my-gcp-project',
                'topic' => 'my-topic',
                'authentication' => 'SERVICE_ACCOUNT',
                'seekType' => 'earliest',
                'serviceAccountKey' => [
                                'serviceAccountFile' => '<string>'
                ],
                'seekTimestamp' => '2026-04-10T12:00:00Z',
                'filter' => '<string>',
                'enableOrdering' => true,
                'ackDeadline' => 305
        ],
        'postgres' => [
                'credentials' => [
                                'username' => 'postgres_user',
                                'password' => 'your_secure_password'
                ],
                'host' => 'my-postgres-server.example.com',
                'port' => 5432,
                'database' => 'production_db',
                'settings' => [
                                'syncIntervalSeconds' => 60,
                                'pullBatchSize' => 1000,
                                'publicationName' => 'clickpipes_publication',
                                'replicationMode' => 'cdc',
                                'replicationSlotName' => 'clickpipes_slot',
                                'allowNullableColumns' => false,
                                'initialLoadParallelism' => 1,
                                'snapshotNumRowsPerPartition' => 100000,
                                'snapshotNumberOfParallelTables' => 1,
                                'enableFailoverSlots' => false,
                                'deleteOnMerge' => false
                ],
                'authentication' => 'IAM_ROLE',
                'iamRole' => 'arn:aws:iam::123456789012:role/MyApplicationRole',
                'tlsHost' => 'my-postgres-server.example.com',
                'caCertificate' => '-----BEGIN CERTIFICATE-----
...',
                'disableTls' => false,
                'skipCertVerification' => false,
                'tableMappings' => [
                                [
                                                                'sourceSchemaName' => 'public',
                                                                'sourceTable' => 'users',
                                                                'targetTable' => 'public_users',
                                                                'excludedColumns' => [
                                                                                                                                'internal_id',
                                                                                                                                'temp_data'
                                                                ],
                                                                'useCustomSortingKey' => false,
                                                                'sortingKeys' => [
                                                                                                                                'created_at_date',
                                                                                                                                'event_id'
                                                                ],
                                                                'tableEngine' => 'ReplacingMergeTree',
                                                                'partitionKey' => 'id'
                                ]
                ]
        ],
        'mysql' => [
                'host' => 'my-mysql-server.example.com',
                'port' => 3306,
                'settings' => [
                                'replicationMode' => 'cdc',
                                'syncIntervalSeconds' => 60,
                                'pullBatchSize' => 1000,
                                'replicationMechanism' => 'GTID',
                                'useCompression' => false,
                                'allowNullableColumns' => false,
                                'initialLoadParallelism' => 1,
                                'snapshotNumRowsPerPartition' => 100000,
                                'snapshotNumberOfParallelTables' => 1,
                                'deleteOnMerge' => false
                ],
                'tableMappings' => [
                                [
                                                                'sourceSchemaName' => 'my_database',
                                                                'sourceTable' => 'users',
                                                                'targetTable' => 'my_database_users',
                                                                'excludedColumns' => [
                                                                                                                                'internal_id',
                                                                                                                                'temp_data'
                                                                ],
                                                                'useCustomSortingKey' => false,
                                                                'sortingKeys' => [
                                                                                                                                'created_at_date',
                                                                                                                                'event_id'
                                                                ],
                                                                'tableEngine' => 'ReplacingMergeTree',
                                                                'partitionKey' => 'id'
                                ]
                ],
                'credentials' => [
                                'username' => 'postgres_user',
                                'password' => 'your_secure_password'
                ],
                'authentication' => 'basic',
                'iamRole' => 'arn:aws:iam::123456789012:role/MyApplicationRole',
                'tlsHost' => 'my-mysql-server.example.com',
                'caCertificate' => '-----BEGIN CERTIFICATE-----
...',
                'disableTls' => false,
                'skipCertVerification' => false,
                'serverId' => 4242
        ],
        'bigquery' => [
                'snapshotStagingPath' => '<string>',
                'settings' => [
                                'replicationMode' => 'snapshot',
                                'allowNullableColumns' => true,
                                'initialLoadParallelism' => 123,
                                'snapshotNumRowsPerPartition' => 123,
                                'snapshotNumberOfParallelTables' => 123
                ],
                'tableMappings' => [
                                [
                                                                'sourceDatasetName' => '<string>',
                                                                'sourceTable' => '<string>',
                                                                'targetTable' => '<string>',
                                                                'excludedColumns' => [
                                                                                                                                '<string>'
                                                                ],
                                                                'useCustomSortingKey' => true,
                                                                'sortingKeys' => [
                                                                                                                                '<string>'
                                                                ]
                                ]
                ],
                'credentials' => [
                                'serviceAccountFile' => '<string>'
                ]
        ],
        'mongodb' => [
                'uri' => 'mongodb+srv://cluster0.example.mongodb.net/mydb',
                'readPreference' => 'secondaryPreferred',
                'settings' => [
                                'replicationMode' => 'cdc',
                                'syncIntervalSeconds' => 60,
                                'pullBatchSize' => 100000,
                                'snapshotNumRowsPerPartition' => 100000,
                                'snapshotNumberOfParallelTables' => 1,
                                'deleteOnMerge' => false,
                                'useJsonNativeFormat' => true
                ],
                'tableMappings' => [
                                [
                                                                'sourceDatabaseName' => 'mydb',
                                                                'sourceCollection' => 'users',
                                                                'targetTable' => 'mydb_users',
                                                                'tableEngine' => 'ReplacingMergeTree'
                                ]
                ],
                'credentials' => [
                                'username' => 'postgres_user',
                                'password' => 'your_secure_password'
                ],
                'tlsHost' => 'cluster0.example.mongodb.net',
                'disableTls' => false,
                'skipCertVerification' => false,
                'caCertificate' => '-----BEGIN CERTIFICATE-----
...'
        ],
        'validateSamples' => true
    ],
    'destination' => [
        'database' => '<string>',
        'table' => '<string>',
        'managedTable' => true,
        'tableDefinition' => [
                'engine' => [
                                'versionColumnId' => '<string>',
                                'columnIds' => [
                                                                '<string>'
                                ]
                ],
                'sortingKey' => [
                                '<string>'
                ],
                'partitionBy' => '<string>',
                'primaryKey' => '<string>'
        ],
        'columns' => [
                [
                                'name' => '<string>',
                                'type' => '<string>'
                ]
        ],
        'roles' => [
                '<string>'
        ]
    ],
    'fieldMappings' => [
        [
                'sourceField' => '<string>',
                'destinationField' => '<string>'
        ]
    ],
    'scaling' => [
        'replicas' => 20,
        'concurrency' => 123,
        'replicaCpuMillicores' => 1062,
        'replicaMemoryGb' => 4.25
    ],
    'settings' => [
        'streaming_max_insert_wait_ms' => 5000,
        'object_storage_concurrency' => 1,
        'object_storage_polling_interval_ms' => 30000,
        'object_storage_max_insert_bytes' => 10737418240,
        'object_storage_max_file_count' => 100,
        'clickhouse_max_threads' => 8,
        'clickhouse_max_insert_threads' => 1,
        'clickhouse_min_insert_block_size_bytes' => 1073741824,
        'clickhouse_max_download_threads' => 4,
        'clickhouse_parallel_distributed_insert_select' => 2,
        'object_storage_use_cluster_function' => true,
        'clickhouse_parallel_view_processing' => false
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Basic <encoded-value>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes"

	payload := strings.NewReader("{\n  \"name\": \"<string>\",\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"objectStorage\": {\n      \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n      \"delimiter\": \",\",\n      \"compression\": \"auto\",\n      \"isContinuous\": true,\n      \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n      \"skipInitialLoad\": true,\n      \"startAfter\": \"events/2026-06-01/\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n      \"path\": \"data/logs/*.json\",\n      \"azureContainerName\": \"mycontainer\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      },\n      \"serviceAccountKey\": \"<string>\"\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    },\n    \"pubsub\": {\n      \"format\": \"JSONEachRow\",\n      \"projectId\": \"my-gcp-project\",\n      \"topic\": \"my-topic\",\n      \"authentication\": \"SERVICE_ACCOUNT\",\n      \"seekType\": \"earliest\",\n      \"serviceAccountKey\": {\n        \"serviceAccountFile\": \"<string>\"\n      },\n      \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n      \"filter\": \"<string>\",\n      \"enableOrdering\": true,\n      \"ackDeadline\": 305\n    },\n    \"postgres\": {\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"host\": \"my-postgres-server.example.com\",\n      \"port\": 5432,\n      \"database\": \"production_db\",\n      \"settings\": {\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"publicationName\": \"clickpipes_publication\",\n        \"replicationMode\": \"cdc\",\n        \"replicationSlotName\": \"clickpipes_slot\",\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"enableFailoverSlots\": false,\n        \"deleteOnMerge\": false\n      },\n      \"authentication\": \"IAM_ROLE\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-postgres-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"public\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"public_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ]\n    },\n    \"mysql\": {\n      \"host\": \"my-mysql-server.example.com\",\n      \"port\": 3306,\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"replicationMechanism\": \"GTID\",\n        \"useCompression\": false,\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false\n      },\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"my_database\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"my_database_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"authentication\": \"basic\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-mysql-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"serverId\": 4242\n    },\n    \"bigquery\": {\n      \"snapshotStagingPath\": \"<string>\",\n      \"settings\": {\n        \"replicationMode\": \"snapshot\",\n        \"allowNullableColumns\": true,\n        \"initialLoadParallelism\": 123,\n        \"snapshotNumRowsPerPartition\": 123,\n        \"snapshotNumberOfParallelTables\": 123\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatasetName\": \"<string>\",\n          \"sourceTable\": \"<string>\",\n          \"targetTable\": \"<string>\",\n          \"excludedColumns\": [\n            \"<string>\"\n          ],\n          \"useCustomSortingKey\": true,\n          \"sortingKeys\": [\n            \"<string>\"\n          ]\n        }\n      ],\n      \"credentials\": {\n        \"serviceAccountFile\": \"<string>\"\n      }\n    },\n    \"mongodb\": {\n      \"uri\": \"mongodb+srv://cluster0.example.mongodb.net/mydb\",\n      \"readPreference\": \"secondaryPreferred\",\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 100000,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false,\n        \"useJsonNativeFormat\": true\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatabaseName\": \"mydb\",\n          \"sourceCollection\": \"users\",\n          \"targetTable\": \"mydb_users\",\n          \"tableEngine\": \"ReplacingMergeTree\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"tlsHost\": \"cluster0.example.mongodb.net\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\"\n    },\n    \"validateSamples\": true\n  },\n  \"destination\": {\n    \"database\": \"<string>\",\n    \"table\": \"<string>\",\n    \"managedTable\": true,\n    \"tableDefinition\": {\n      \"engine\": {\n        \"versionColumnId\": \"<string>\",\n        \"columnIds\": [\n          \"<string>\"\n        ]\n      },\n      \"sortingKey\": [\n        \"<string>\"\n      ],\n      \"partitionBy\": \"<string>\",\n      \"primaryKey\": \"<string>\"\n    },\n    \"columns\": [\n      {\n        \"name\": \"<string>\",\n        \"type\": \"<string>\"\n      }\n    ],\n    \"roles\": [\n      \"<string>\"\n    ]\n  },\n  \"fieldMappings\": [\n    {\n      \"sourceField\": \"<string>\",\n      \"destinationField\": \"<string>\"\n    }\n  ],\n  \"scaling\": {\n    \"replicas\": 20,\n    \"concurrency\": 123,\n    \"replicaCpuMillicores\": 1062,\n    \"replicaMemoryGb\": 4.25\n  },\n  \"settings\": {\n    \"streaming_max_insert_wait_ms\": 5000,\n    \"object_storage_concurrency\": 1,\n    \"object_storage_polling_interval_ms\": 30000,\n    \"object_storage_max_insert_bytes\": 10737418240,\n    \"object_storage_max_file_count\": 100,\n    \"clickhouse_max_threads\": 8,\n    \"clickhouse_max_insert_threads\": 1,\n    \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n    \"clickhouse_max_download_threads\": 4,\n    \"clickhouse_parallel_distributed_insert_select\": 2,\n    \"object_storage_use_cluster_function\": true,\n    \"clickhouse_parallel_view_processing\": false\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Basic <encoded-value>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")
  .header("Authorization", "Basic <encoded-value>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"<string>\",\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"objectStorage\": {\n      \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n      \"delimiter\": \",\",\n      \"compression\": \"auto\",\n      \"isContinuous\": true,\n      \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n      \"skipInitialLoad\": true,\n      \"startAfter\": \"events/2026-06-01/\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n      \"path\": \"data/logs/*.json\",\n      \"azureContainerName\": \"mycontainer\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      },\n      \"serviceAccountKey\": \"<string>\"\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    },\n    \"pubsub\": {\n      \"format\": \"JSONEachRow\",\n      \"projectId\": \"my-gcp-project\",\n      \"topic\": \"my-topic\",\n      \"authentication\": \"SERVICE_ACCOUNT\",\n      \"seekType\": \"earliest\",\n      \"serviceAccountKey\": {\n        \"serviceAccountFile\": \"<string>\"\n      },\n      \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n      \"filter\": \"<string>\",\n      \"enableOrdering\": true,\n      \"ackDeadline\": 305\n    },\n    \"postgres\": {\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"host\": \"my-postgres-server.example.com\",\n      \"port\": 5432,\n      \"database\": \"production_db\",\n      \"settings\": {\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"publicationName\": \"clickpipes_publication\",\n        \"replicationMode\": \"cdc\",\n        \"replicationSlotName\": \"clickpipes_slot\",\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"enableFailoverSlots\": false,\n        \"deleteOnMerge\": false\n      },\n      \"authentication\": \"IAM_ROLE\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-postgres-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"public\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"public_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ]\n    },\n    \"mysql\": {\n      \"host\": \"my-mysql-server.example.com\",\n      \"port\": 3306,\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"replicationMechanism\": \"GTID\",\n        \"useCompression\": false,\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false\n      },\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"my_database\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"my_database_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"authentication\": \"basic\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-mysql-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"serverId\": 4242\n    },\n    \"bigquery\": {\n      \"snapshotStagingPath\": \"<string>\",\n      \"settings\": {\n        \"replicationMode\": \"snapshot\",\n        \"allowNullableColumns\": true,\n        \"initialLoadParallelism\": 123,\n        \"snapshotNumRowsPerPartition\": 123,\n        \"snapshotNumberOfParallelTables\": 123\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatasetName\": \"<string>\",\n          \"sourceTable\": \"<string>\",\n          \"targetTable\": \"<string>\",\n          \"excludedColumns\": [\n            \"<string>\"\n          ],\n          \"useCustomSortingKey\": true,\n          \"sortingKeys\": [\n            \"<string>\"\n          ]\n        }\n      ],\n      \"credentials\": {\n        \"serviceAccountFile\": \"<string>\"\n      }\n    },\n    \"mongodb\": {\n      \"uri\": \"mongodb+srv://cluster0.example.mongodb.net/mydb\",\n      \"readPreference\": \"secondaryPreferred\",\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 100000,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false,\n        \"useJsonNativeFormat\": true\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatabaseName\": \"mydb\",\n          \"sourceCollection\": \"users\",\n          \"targetTable\": \"mydb_users\",\n          \"tableEngine\": \"ReplacingMergeTree\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"tlsHost\": \"cluster0.example.mongodb.net\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\"\n    },\n    \"validateSamples\": true\n  },\n  \"destination\": {\n    \"database\": \"<string>\",\n    \"table\": \"<string>\",\n    \"managedTable\": true,\n    \"tableDefinition\": {\n      \"engine\": {\n        \"versionColumnId\": \"<string>\",\n        \"columnIds\": [\n          \"<string>\"\n        ]\n      },\n      \"sortingKey\": [\n        \"<string>\"\n      ],\n      \"partitionBy\": \"<string>\",\n      \"primaryKey\": \"<string>\"\n    },\n    \"columns\": [\n      {\n        \"name\": \"<string>\",\n        \"type\": \"<string>\"\n      }\n    ],\n    \"roles\": [\n      \"<string>\"\n    ]\n  },\n  \"fieldMappings\": [\n    {\n      \"sourceField\": \"<string>\",\n      \"destinationField\": \"<string>\"\n    }\n  ],\n  \"scaling\": {\n    \"replicas\": 20,\n    \"concurrency\": 123,\n    \"replicaCpuMillicores\": 1062,\n    \"replicaMemoryGb\": 4.25\n  },\n  \"settings\": {\n    \"streaming_max_insert_wait_ms\": 5000,\n    \"object_storage_concurrency\": 1,\n    \"object_storage_polling_interval_ms\": 30000,\n    \"object_storage_max_insert_bytes\": 10737418240,\n    \"object_storage_max_file_count\": 100,\n    \"clickhouse_max_threads\": 8,\n    \"clickhouse_max_insert_threads\": 1,\n    \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n    \"clickhouse_max_download_threads\": 4,\n    \"clickhouse_parallel_distributed_insert_select\": 2,\n    \"object_storage_use_cluster_function\": true,\n    \"clickhouse_parallel_view_processing\": false\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Basic <encoded-value>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"<string>\",\n  \"source\": {\n    \"kafka\": {\n      \"brokers\": \"<string>\",\n      \"topics\": \"<string>\",\n      \"consumerGroup\": \"my-clickpipe-consumer-group\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"offset\": {\n        \"timestamp\": \"2021-01-01T00:00\"\n      },\n      \"schemaRegistry\": {\n        \"url\": \"https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004\",\n        \"authentication\": \"PLAIN\",\n        \"caCertificate\": \"<string>\",\n        \"credentials\": {\n          \"username\": \"<string>\",\n          \"password\": \"<string>\"\n        }\n      },\n      \"caCertificate\": \"<string>\",\n      \"reversePrivateEndpointIds\": [\n        \"<string>\"\n      ],\n      \"exactlyOnce\": true,\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      }\n    },\n    \"objectStorage\": {\n      \"url\": \"https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz\",\n      \"delimiter\": \",\",\n      \"compression\": \"auto\",\n      \"isContinuous\": true,\n      \"queueUrl\": \"https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue\",\n      \"skipInitialLoad\": true,\n      \"startAfter\": \"events/2026-06-01/\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"connectionString\": \"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net\",\n      \"path\": \"data/logs/*.json\",\n      \"azureContainerName\": \"mycontainer\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      },\n      \"serviceAccountKey\": \"<string>\"\n    },\n    \"kinesis\": {\n      \"streamName\": \"my-stream\",\n      \"region\": \"us-east-1\",\n      \"useEnhancedFanOut\": true,\n      \"timestamp\": 1615766400,\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyRole\",\n      \"accessKey\": {\n        \"accessKeyId\": \"<string>\",\n        \"secretKey\": \"<string>\"\n      }\n    },\n    \"pubsub\": {\n      \"format\": \"JSONEachRow\",\n      \"projectId\": \"my-gcp-project\",\n      \"topic\": \"my-topic\",\n      \"authentication\": \"SERVICE_ACCOUNT\",\n      \"seekType\": \"earliest\",\n      \"serviceAccountKey\": {\n        \"serviceAccountFile\": \"<string>\"\n      },\n      \"seekTimestamp\": \"2026-04-10T12:00:00Z\",\n      \"filter\": \"<string>\",\n      \"enableOrdering\": true,\n      \"ackDeadline\": 305\n    },\n    \"postgres\": {\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"host\": \"my-postgres-server.example.com\",\n      \"port\": 5432,\n      \"database\": \"production_db\",\n      \"settings\": {\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"publicationName\": \"clickpipes_publication\",\n        \"replicationMode\": \"cdc\",\n        \"replicationSlotName\": \"clickpipes_slot\",\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"enableFailoverSlots\": false,\n        \"deleteOnMerge\": false\n      },\n      \"authentication\": \"IAM_ROLE\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-postgres-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"public\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"public_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ]\n    },\n    \"mysql\": {\n      \"host\": \"my-mysql-server.example.com\",\n      \"port\": 3306,\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 1000,\n        \"replicationMechanism\": \"GTID\",\n        \"useCompression\": false,\n        \"allowNullableColumns\": false,\n        \"initialLoadParallelism\": 1,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false\n      },\n      \"tableMappings\": [\n        {\n          \"sourceSchemaName\": \"my_database\",\n          \"sourceTable\": \"users\",\n          \"targetTable\": \"my_database_users\",\n          \"excludedColumns\": [\n            \"internal_id\",\n            \"temp_data\"\n          ],\n          \"useCustomSortingKey\": false,\n          \"sortingKeys\": [\n            \"created_at_date\",\n            \"event_id\"\n          ],\n          \"tableEngine\": \"ReplacingMergeTree\",\n          \"partitionKey\": \"id\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"authentication\": \"basic\",\n      \"iamRole\": \"arn:aws:iam::123456789012:role/MyApplicationRole\",\n      \"tlsHost\": \"my-mysql-server.example.com\",\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"serverId\": 4242\n    },\n    \"bigquery\": {\n      \"snapshotStagingPath\": \"<string>\",\n      \"settings\": {\n        \"replicationMode\": \"snapshot\",\n        \"allowNullableColumns\": true,\n        \"initialLoadParallelism\": 123,\n        \"snapshotNumRowsPerPartition\": 123,\n        \"snapshotNumberOfParallelTables\": 123\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatasetName\": \"<string>\",\n          \"sourceTable\": \"<string>\",\n          \"targetTable\": \"<string>\",\n          \"excludedColumns\": [\n            \"<string>\"\n          ],\n          \"useCustomSortingKey\": true,\n          \"sortingKeys\": [\n            \"<string>\"\n          ]\n        }\n      ],\n      \"credentials\": {\n        \"serviceAccountFile\": \"<string>\"\n      }\n    },\n    \"mongodb\": {\n      \"uri\": \"mongodb+srv://cluster0.example.mongodb.net/mydb\",\n      \"readPreference\": \"secondaryPreferred\",\n      \"settings\": {\n        \"replicationMode\": \"cdc\",\n        \"syncIntervalSeconds\": 60,\n        \"pullBatchSize\": 100000,\n        \"snapshotNumRowsPerPartition\": 100000,\n        \"snapshotNumberOfParallelTables\": 1,\n        \"deleteOnMerge\": false,\n        \"useJsonNativeFormat\": true\n      },\n      \"tableMappings\": [\n        {\n          \"sourceDatabaseName\": \"mydb\",\n          \"sourceCollection\": \"users\",\n          \"targetTable\": \"mydb_users\",\n          \"tableEngine\": \"ReplacingMergeTree\"\n        }\n      ],\n      \"credentials\": {\n        \"username\": \"postgres_user\",\n        \"password\": \"your_secure_password\"\n      },\n      \"tlsHost\": \"cluster0.example.mongodb.net\",\n      \"disableTls\": false,\n      \"skipCertVerification\": false,\n      \"caCertificate\": \"-----BEGIN CERTIFICATE-----\\n...\"\n    },\n    \"validateSamples\": true\n  },\n  \"destination\": {\n    \"database\": \"<string>\",\n    \"table\": \"<string>\",\n    \"managedTable\": true,\n    \"tableDefinition\": {\n      \"engine\": {\n        \"versionColumnId\": \"<string>\",\n        \"columnIds\": [\n          \"<string>\"\n        ]\n      },\n      \"sortingKey\": [\n        \"<string>\"\n      ],\n      \"partitionBy\": \"<string>\",\n      \"primaryKey\": \"<string>\"\n    },\n    \"columns\": [\n      {\n        \"name\": \"<string>\",\n        \"type\": \"<string>\"\n      }\n    ],\n    \"roles\": [\n      \"<string>\"\n    ]\n  },\n  \"fieldMappings\": [\n    {\n      \"sourceField\": \"<string>\",\n      \"destinationField\": \"<string>\"\n    }\n  ],\n  \"scaling\": {\n    \"replicas\": 20,\n    \"concurrency\": 123,\n    \"replicaCpuMillicores\": 1062,\n    \"replicaMemoryGb\": 4.25\n  },\n  \"settings\": {\n    \"streaming_max_insert_wait_ms\": 5000,\n    \"object_storage_concurrency\": 1,\n    \"object_storage_polling_interval_ms\": 30000,\n    \"object_storage_max_insert_bytes\": 10737418240,\n    \"object_storage_max_file_count\": 100,\n    \"clickhouse_max_threads\": 8,\n    \"clickhouse_max_insert_threads\": 1,\n    \"clickhouse_min_insert_block_size_bytes\": 1073741824,\n    \"clickhouse_max_download_threads\": 4,\n    \"clickhouse_parallel_distributed_insert_select\": 2,\n    \"object_storage_use_cluster_function\": true,\n    \"clickhouse_parallel_view_processing\": false\n  }\n}"

response = http.request(request)
puts response.read_body