From c7931ab806f2f1796bdf18eabaf0589ea13c0389 Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:10:17 -0600 Subject: [PATCH 1/8] feat(node): add ClickHouse instrumentation --- .../node-integration-tests/package.json | 1 + .../tracing/clickhouse/docker-compose.yml | 20 ++ .../suites/tracing/clickhouse/instrument.mjs | 9 + .../suites/tracing/clickhouse/scenario.mjs | 61 ++++ .../suites/tracing/clickhouse/test.ts | 116 +++++++ packages/astro/src/index.server.ts | 1 + packages/aws-serverless/src/index.ts | 1 + packages/bun/src/index.ts | 1 + packages/cloudflare/src/index.ts | 1 + packages/deno/src/index.ts | 1 + packages/elysia/src/index.ts | 1 + packages/google-cloud-serverless/src/index.ts | 1 + packages/node/src/index.ts | 1 + packages/server-utils/src/index.ts | 1 + .../src/integrations/clickhouse.ts | 155 +++++++++ .../server-utils/src/integrations/index.ts | 2 + .../server-utils/src/orchestrion/channels.ts | 2 + .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/clickhouse.ts | 44 +++ .../src/orchestrion/config/index.ts | 2 + .../server-utils/src/utils/sql-sanitizer.ts | 296 ++++++++++++++++++ packages/server-utils/src/utils/sql.ts | 242 +------------- .../test/integrations/clickhouse.test.ts | 256 +++++++++++++++ packages/server-utils/test/utils/sql.test.ts | 36 +++ packages/solidstart/src/server/index.ts | 1 + packages/sveltekit/src/server/index.ts | 1 + pnpm-lock.yaml | 9 + 27 files changed, 1026 insertions(+), 237 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/clickhouse/docker-compose.yml create mode 100644 dev-packages/node-integration-tests/suites/tracing/clickhouse/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts create mode 100644 packages/server-utils/src/integrations/clickhouse.ts create mode 100644 packages/server-utils/src/orchestrion/config/clickhouse.ts create mode 100644 packages/server-utils/src/utils/sql-sanitizer.ts create mode 100644 packages/server-utils/test/integrations/clickhouse.test.ts diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index 84014b7c29f8..ad59e88d7be7 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -35,6 +35,7 @@ "@aws-sdk/client-sfn": "^3.1161.0", "@aws-sdk/client-sns": "^3.1161.0", "@aws-sdk/client-sqs": "^3.1161.0", + "@clickhouse/client": "1.25.0", "@earendil-works/chord": "^1.4.0", "@earendil-works/pi-ai": "^1.4.0", "@earendil-works/pi-durable": "^1.4.0", diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/docker-compose.yml b/dev-packages/node-integration-tests/suites/tracing/clickhouse/docker-compose.yml new file mode 100644 index 000000000000..dfc23f3c1827 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/docker-compose.yml @@ -0,0 +1,20 @@ +services: + db: + image: clickhouse/clickhouse-server:25.8 + restart: always + ports: + - '8123:8123' + environment: + CLICKHOUSE_DB: tests + CLICKHOUSE_USER: test + CLICKHOUSE_PASSWORD: test + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: ['CMD-SHELL', 'clickhouse-client --user test --password test --query "SELECT 1"'] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/clickhouse/instrument.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/instrument.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs new file mode 100644 index 000000000000..01fc63ba13ae --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs @@ -0,0 +1,61 @@ +import * as Sentry from '@sentry/node'; +import { waitForConnection } from '@sentry-internal/node-integration-tests'; +import { createClient } from '@clickhouse/client'; + +const client = createClient({ + url: 'http://localhost:8123', + username: 'test', + password: 'test', + database: 'tests', +}); + +async function run() { + // `ping` isn't instrumented, so the probe adds no spans. + await waitForConnection(async () => { + const result = await client.ping(); + if (!result.success) { + throw result.error; + } + }); + + await Sentry.startSpan({ name: 'Test Span', op: 'test' }, async () => { + try { + await client.command({ + query: 'CREATE TABLE IF NOT EXISTS events (id UInt32, name String) ENGINE = MergeTree ORDER BY id', + }); + + await client.insert({ table: 'events', values: [{ id: 1, name: 'signup' }], format: 'JSONEachRow' }); + await client.command({ query: 'INSERT INTO events FORMAT CSV\n2,inline-sensitive@example.com' }); + + const literal = await client.query({ + query: "SELECT name FROM events WHERE name = 'signup'", + format: 'JSONEachRow', + }); + await literal.json(); + + const parameterized = await client.query({ + query: 'SELECT name FROM events WHERE id = {id:UInt32}', + query_params: { id: 1 }, + format: 'JSONEachRow', + }); + await parameterized.json(); + + const { stream } = await client.exec({ query: 'SELECT count() FROM events FORMAT JSONEachRow' }); + for await (const _chunk of stream) { + // drain the response so the socket is released + } + + // Failed calls the caller handles: each should leave an errored span and no error event. + await client.query({ query: 'SELECT * FROM does_not_exist' }).catch(() => {}); + await client.command({ query: 'DROP TABLE does_not_exist' }).catch(() => {}); + await client.exec({ query: 'SELECT * FROM does_not_exist FORMAT JSONEachRow' }).catch(() => {}); + await client.insert({ table: 'does_not_exist', values: [{ id: 1 }], format: 'JSONEachRow' }).catch(() => {}); + } finally { + await client.command({ query: 'DROP TABLE IF EXISTS events' }); + } + }); + + await client.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts new file mode 100644 index 000000000000..d91dcf29bf39 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts @@ -0,0 +1,116 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; +import { + DB_COLLECTION_NAME, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; +import { afterAll, describe, expect } from 'vitest'; +import { streamedAttribute as attr } from '../../../utils'; +import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose } from '../../../utils/runner'; + +const ORIGIN = 'auto.db.clickhouse'; + +const CONNECTION_ATTRIBUTES = { + [SENTRY_OP]: attr(DB), + [SENTRY_KIND]: attr('client'), + [SENTRY_ORIGIN]: attr(ORIGIN), + [DB_SYSTEM_NAME]: attr('clickhouse'), + [DB_NAMESPACE]: attr('tests'), + [DB_USER]: attr('test'), + [SERVER_ADDRESS]: attr('localhost'), + [SERVER_PORT]: attr(8123, 'integer'), +}; + +const CREATE_TABLE = 'CREATE TABLE IF NOT EXISTS events (id UInt32, name String) ENGINE = MergeTree ORDER BY id'; + +// One entry per instrumented call, in the order the scenario makes them. +const EXPECTED_SPANS = [ + { name: 'CREATE TABLE events', status: 'ok', operation: 'CREATE', text: CREATE_TABLE }, + { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: undefined, collection: 'events' }, + { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: 'INSERT INTO events' }, + // the string literal is sanitized out of the statement + { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT name FROM events WHERE name = ?' }, + // `{name:Type}` query parameters are kept as written + { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT name FROM events WHERE id = {id:UInt32}' }, + { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT count() FROM events FORMAT JSONEachRow' }, + // handled failures, one per method + { name: 'SELECT does_not_exist', status: 'error', operation: 'SELECT', text: 'SELECT * FROM does_not_exist' }, + { name: 'DROP TABLE does_not_exist', status: 'error', operation: 'DROP', text: 'DROP TABLE does_not_exist' }, + { + name: 'SELECT does_not_exist', + status: 'error', + operation: 'SELECT', + text: 'SELECT * FROM does_not_exist FORMAT JSONEachRow', + }, + { + name: 'INSERT does_not_exist', + status: 'error', + operation: 'INSERT', + text: undefined, + collection: 'does_not_exist', + }, + { name: 'DROP TABLE events', status: 'ok', operation: 'DROP', text: 'DROP TABLE IF EXISTS events' }, +]; + +function assertClickhouseSpans(container: SerializedStreamedSpanContainer): void { + const segment = container.items.find(span => span.is_segment); + expect(segment?.name).toBe('Test Span'); + + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === ORIGIN); + for (const span of spans) { + expect(span.parent_span_id).toBe(segment?.span_id); + expect(span.attributes).toEqual(expect.objectContaining(CONNECTION_ATTRIBUTES)); + } + + expect( + spans.map(span => ({ + name: span.name, + status: span.status, + operation: span.attributes[DB_OPERATION_NAME]?.value, + text: span.attributes[DB_QUERY_TEXT]?.value, + ...(span.attributes[DB_COLLECTION_NAME] ? { collection: span.attributes[DB_COLLECTION_NAME].value } : {}), + })), + ).toEqual(EXPECTED_SPANS); + expect(spans.map(span => span.attributes[DB_QUERY_SUMMARY]?.value)).toEqual(EXPECTED_SPANS.map(span => span.name)); +} + +describeWithDockerCompose('clickhouse auto instrumentation', { workingDirectory: [__dirname] }, () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + // The span container is the only envelope expected. A captured error would arrive as an `event` + // envelope ahead of it and fail the run, so this also checks that handled failures aren't captured. + describe('@clickhouse/client >=1.23', () => { + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createTestRunner, test) => { + test('instruments `query`, `command`, `exec` and `insert`', { timeout: 90_000 }, async () => { + await createTestRunner().expect({ span: assertClickhouseSpans }).start().completed(); + }); + }); + }); + + // Before 1.23.0 the client lives in the separate `@clickhouse/client-common` package. + describe('@clickhouse/client <1.23', () => { + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createTestRunner, test) => { + test('instruments `query`, `command`, `exec` and `insert`', { timeout: 90_000 }, async () => { + await createTestRunner().expect({ span: assertClickhouseSpans }).start().completed(); + }); + }, + { additionalDependencies: { '@clickhouse/client': '1.22.0' } }, + ); + }); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index fde9310382cd..37fa15243c9d 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -120,6 +120,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, openTelemetryIntegration, getOtlpTracesEndpoint, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index 3150e1faf053..3d8b79f8d7ab 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -139,6 +139,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, processSessionIntegration, prismaIntegration, openTelemetryIntegration, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 1fb4be3e0d98..5c280f00bae4 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -162,6 +162,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, openTelemetryIntegration, getOtlpTracesEndpoint, diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 626f0531d5e4..b86371145050 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -192,6 +192,7 @@ export { mysql2Integration, mysqlIntegration, neonIntegration, + clickhouseIntegration, nitroIntegration, openAIIntegration, piAiIntegration, diff --git a/packages/deno/src/index.ts b/packages/deno/src/index.ts index 9c2a66ad7da4..50a818d3a006 100644 --- a/packages/deno/src/index.ts +++ b/packages/deno/src/index.ts @@ -200,6 +200,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, cronJobIntegration, nodeCronIntegration, nodeScheduleIntegration, diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index b26b2409106f..3e595fe76d4e 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -135,6 +135,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, processSessionIntegration, hapiIntegration, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index 4f1f89b10d05..df6a6bd3f97c 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -139,6 +139,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, openTelemetryIntegration, getOtlpTracesEndpoint, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 3efa0e3b325a..2df9cd0791bf 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -53,6 +53,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, redisIntegration, tediousIntegration, vercelAIIntegration, diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 71f23effdaae..f44baa230581 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -86,6 +86,7 @@ export { openAIIntegration } from './integrations/openai'; export { postgresIntegration } from './integrations/pg'; export { postgresJsIntegration } from './integrations/postgresjs'; export { neonIntegration } from './integrations/neon'; +export { clickhouseIntegration } from './integrations/clickhouse'; export { tediousIntegration } from './integrations/tedious'; export { vercelAIIntegration } from './integrations/vercel-ai'; export { expressIntegration } from './integrations/express'; diff --git a/packages/server-utils/src/integrations/clickhouse.ts b/packages/server-utils/src/integrations/clickhouse.ts new file mode 100644 index 000000000000..bb3e5232caff --- /dev/null +++ b/packages/server-utils/src/integrations/clickhouse.ts @@ -0,0 +1,155 @@ +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; +import { + DB_COLLECTION_NAME, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY_SUMMARY, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; +import type { IntegrationFn, SpanAttributes } from '@sentry/core'; +import { defineIntegration, startInactiveSpan } from '@sentry/core'; +import type { OrchestrionChannelContext } from '../orchestrion/types'; +import { CHANNELS } from '../orchestrion/channels'; +import { clickhouseModuleNames } from '../orchestrion/config/clickhouse'; +import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +import { bindTracingChannelToSpan } from '../tracing-channel'; +import { getLowCardinalitySqlSpanName, getSqlQueryAttributes } from '../utils/sql'; + +const INTEGRATION_NAME = 'ClickHouse' as const; +const ORIGIN = 'auto.db.clickhouse'; +const DB_SYSTEM_CLICKHOUSE = 'clickhouse'; + +type ClickHouseMethod = 'query' | 'command' | 'exec' | 'insert'; + +// The parts of `ClickHouseClient`'s `connectionParams` the span reads. `>=1.10` moved the username +// into `auth` (which can hold an access token instead); older versions keep it at the top level. +interface ClickHouseConnectionParams { + url?: URL; + database?: string; + username?: string; + auth?: { username?: string }; +} + +interface ClickHouseClient { + connectionParams?: ClickHouseConnectionParams; +} + +// `query`/`command`/`exec` take `{ query }`, `insert` takes `{ table, values }`. Every method accepts a +// per-call `auth` override. +interface ClickHouseParams { + query?: unknown; + table?: unknown; + values?: unknown; + auth?: { username?: unknown }; +} + +/** Context orchestrion attaches to the channels wrapping `ClickHouseClient`'s methods. */ +interface ClickHouseChannelContext extends OrchestrionChannelContext { + arguments: [ClickHouseParams?, ...unknown[]]; + self?: ClickHouseClient; +} + +const _clickhouseIntegration = (() => { + return { + name: INTEGRATION_NAME, + setup(client) { + invokeOrchestrionInstrumentation(client, clickhouseModuleNames, instrumentClickhouse, []); + }, + }; +}) satisfies IntegrationFn; + +function instrumentClickhouse(): void { + // `@clickhouse/client-common` (`<1.23.0`) and `@clickhouse/client` (`>=1.23.0`) publish the same + // calls under their own module names. + const channels: [string, ClickHouseMethod][] = [ + [CHANNELS.CLICKHOUSE_QUERY, 'query'], + [CHANNELS.CLICKHOUSE_COMMAND, 'command'], + [CHANNELS.CLICKHOUSE_EXEC, 'exec'], + [CHANNELS.CLICKHOUSE_INSERT, 'insert'], + [CHANNELS.CLICKHOUSE_COMMON_QUERY, 'query'], + [CHANNELS.CLICKHOUSE_COMMON_COMMAND, 'command'], + [CHANNELS.CLICKHOUSE_COMMON_EXEC, 'exec'], + [CHANNELS.CLICKHOUSE_COMMON_INSERT, 'insert'], + ]; + + for (const [channelName, method] of channels) { + // Failed calls only get an error status on the span. The error still rejects to the caller, who + // decides whether to handle it, so it isn't captured here. + bindTracingChannelToSpan(diagnosticsChannel.tracingChannel(channelName), data => { + const params = data.arguments[0]; + // `insert` with an empty array returns without sending a request. + if (method === 'insert' && Array.isArray(params?.values) && params.values.length === 0) { + return undefined; + } + + const attributes: SpanAttributes = { + [SENTRY_OP]: DB, + [SENTRY_KIND]: 'client', + [SENTRY_ORIGIN]: ORIGIN, + ...getConnectionAttributes(data.self?.connectionParams, params), + ...(method === 'insert' ? getInsertAttributes(params) : getStatementAttributes(params)), + }; + + return startInactiveSpan({ + name: getLowCardinalitySqlSpanName(attributes, DB_SYSTEM_CLICKHOUSE), + attributes, + onlyIfParent: true, + }); + }); + } +} + +function getStatementAttributes(params: ClickHouseParams | undefined): SpanAttributes { + const query = params?.query; + return { ...getSqlQueryAttributes(typeof query === 'string' ? query : undefined, 'clickhouse') }; +} + +// `insert` builds its `INSERT INTO FORMAT ` statement internally, so the span is +// described from the target table instead of a statement. +function getInsertAttributes(params: ClickHouseParams | undefined): SpanAttributes { + const table = typeof params?.table === 'string' && params.table ? params.table : undefined; + return { + [DB_OPERATION_NAME]: 'INSERT', + [DB_COLLECTION_NAME]: table, + [DB_QUERY_SUMMARY]: table ? `INSERT ${table}` : 'INSERT', + }; +} + +function getConnectionAttributes( + connectionParams: ClickHouseConnectionParams | undefined, + params: ClickHouseParams | undefined, +): SpanAttributes { + const url = connectionParams?.url; + const user = params?.auth?.username ?? connectionParams?.auth?.username ?? connectionParams?.username; + return { + [DB_SYSTEM_NAME]: DB_SYSTEM_CLICKHOUSE, + [DB_NAMESPACE]: connectionParams?.database, + [DB_USER]: typeof user === 'string' ? user : undefined, + [SERVER_ADDRESS]: url?.hostname || undefined, + [SERVER_PORT]: url ? getServerPort(url) : undefined, + }; +} + +// The client talks HTTP(S), so a URL without an explicit port uses the protocol's default. +function getServerPort(url: URL): number { + if (url.port) { + return Number(url.port); + } + return url.protocol === 'https:' ? 443 : 80; +} + +/** + * Diagnostics-channel-based `@clickhouse/client` integration. + * + * Subscribes to the channels Sentry's code transform injects into `ClickHouseClient`'s `query`, + * `command`, `exec` and `insert` methods, and emits a `db` span for each call. Requires the Sentry + * runtime hook or bundler plugin to be active. + */ +export const clickhouseIntegration = defineIntegration(_clickhouseIntegration); diff --git a/packages/server-utils/src/integrations/index.ts b/packages/server-utils/src/integrations/index.ts index 11c90eb7855c..9880cd6a9b81 100644 --- a/packages/server-utils/src/integrations/index.ts +++ b/packages/server-utils/src/integrations/index.ts @@ -29,6 +29,7 @@ import { togetherAIIntegration } from './together-ai'; import { typesafeIntegration } from './typesafe'; import { postgresJsIntegration } from './postgresjs'; import { neonIntegration } from './neon'; +import { clickhouseIntegration } from './clickhouse'; import { firebaseIntegration } from './firebase'; import { expressIntegration } from './express'; import { fastifyIntegration } from './fastify'; @@ -75,6 +76,7 @@ export function getTracingIntegrations(): Integration[] { typesafeIntegration(), postgresJsIntegration(), neonIntegration(), + clickhouseIntegration(), firebaseIntegration(), mcpServerIntegration(), ]; diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index 44d91bbaef28..9456854e1201 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -1,6 +1,7 @@ import { amqplibChannels } from './config/amqplib'; import { anthropicAiChannels } from './config/anthropic-ai'; import { awsSdkChannels } from './config/aws-sdk'; +import { clickhouseChannels } from './config/clickhouse'; import { cronChannels } from './config/cron'; import { dataloaderChannels } from './config/dataloader'; import { expressChannels } from './config/express'; @@ -62,6 +63,7 @@ export const CHANNELS = { ...amqplibChannels, ...anthropicAiChannels, ...awsSdkChannels, + ...clickhouseChannels, ...cronChannels, ...dataloaderChannels, ...expressChannels, diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 635734bf7cdf..353f924e49d9 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -19,6 +19,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'postgresIntegration', modules: ['pg', 'pg-pool'] }, { exportName: 'postgresJsIntegration', modules: ['postgres'] }, { exportName: 'neonIntegration', modules: ['@neondatabase/serverless'] }, + { exportName: 'clickhouseIntegration', modules: ['@clickhouse/client', '@clickhouse/client-common'] }, { exportName: 'prismaIntegration', modules: ['@prisma/orm-family-sql'] }, { exportName: 'mysqlIntegration', modules: ['mysql'] }, { exportName: 'mysql2Integration', modules: ['mysql2'] }, diff --git a/packages/server-utils/src/orchestrion/config/clickhouse.ts b/packages/server-utils/src/orchestrion/config/clickhouse.ts new file mode 100644 index 000000000000..fe0953f914f1 --- /dev/null +++ b/packages/server-utils/src/orchestrion/config/clickhouse.ts @@ -0,0 +1,44 @@ +import type { InstrumentationConfig } from '../apmTypes'; + +import { getModuleNames } from './module-names'; + +// `ClickHouseClient` sends every statement through four `async` methods: `query`, `command`, `exec` +// and `insert`. None of them calls another, so each call is traced as exactly one db span. `ping` +// and `close` carry no statement and are left alone. +// +// Two file layouts across the supported range: +// - `< 1.23.0` → the class lives in the separate `@clickhouse/client-common` package (`dist/client.js`) +// - `>= 1.23.0` → `@clickhouse/client` ships that code itself, in `dist/common/client.js` +// +// Starts at 1.1.0: before it, `command` delegated to `exec` (two spans per command) and the client +// did not keep the `connectionParams` the span attributes are read from. +// +// The Node.js `NodeClickHouseClient` subclass only forwards `query` to `super`, and its class name +// doesn't match `className`, so it doesn't add a second span. +const METHODS = ['query', 'command', 'exec', 'insert'] as const; + +const MODULES = [ + { name: '@clickhouse/client-common', versionRange: '>=1.1.0 <1.23.0', filePath: 'dist/client.js' }, + { name: '@clickhouse/client', versionRange: '>=1.23.0 <2', filePath: 'dist/common/client.js' }, +]; + +export const clickhouseConfig: InstrumentationConfig[] = MODULES.flatMap(module => + METHODS.map(methodName => ({ + channelName: methodName, + module, + functionQuery: { className: 'ClickHouseClient', methodName, kind: 'Async' as const }, + })), +); + +export const clickhouseModuleNames = getModuleNames(clickhouseConfig); + +export const clickhouseChannels = { + CLICKHOUSE_QUERY: 'orchestrion:@clickhouse/client:query', + CLICKHOUSE_COMMAND: 'orchestrion:@clickhouse/client:command', + CLICKHOUSE_EXEC: 'orchestrion:@clickhouse/client:exec', + CLICKHOUSE_INSERT: 'orchestrion:@clickhouse/client:insert', + CLICKHOUSE_COMMON_QUERY: 'orchestrion:@clickhouse/client-common:query', + CLICKHOUSE_COMMON_COMMAND: 'orchestrion:@clickhouse/client-common:command', + CLICKHOUSE_COMMON_EXEC: 'orchestrion:@clickhouse/client-common:exec', + CLICKHOUSE_COMMON_INSERT: 'orchestrion:@clickhouse/client-common:insert', +} as const; diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index 6c5b11dc1d5e..9ef0d96ade50 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -5,6 +5,7 @@ import { uniq } from '@sentry/core'; import { awsSdkConfig } from './aws-sdk'; import { amqplibConfig } from './amqplib'; import { anthropicAiConfig } from './anthropic-ai'; +import { clickhouseConfig } from './clickhouse'; import { cronConfig } from './cron'; import { dataloaderConfig } from './dataloader'; import { expressConfig } from './express'; @@ -68,6 +69,7 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...amqplibConfig, ...anthropicAiConfig, ...awsSdkConfig, + ...clickhouseConfig, ...cronConfig, ...dataloaderConfig, ...expressConfig, diff --git a/packages/server-utils/src/utils/sql-sanitizer.ts b/packages/server-utils/src/utils/sql-sanitizer.ts new file mode 100644 index 000000000000..613a04b228bd --- /dev/null +++ b/packages/server-utils/src/utils/sql-sanitizer.ts @@ -0,0 +1,296 @@ +let integerLiteralRE: RegExp | undefined; +let mssqlIntegerLiteralRE: RegExp | undefined; + +/** + * The lookbehind keeps `$n` and `?n` parameter placeholders out of the match. T-SQL has neither, + * and `$1000` there is a money literal, so `$` only guards a placeholder outside `mssql`. + * + * Lazy init: constructing a lookbehind at module scope evaluates it on import and crashes + * Safari <16.4 browser bundles that reach this file via the core barrel. + */ +function getIntegerLiteralRE(dialect: SqlDialect): RegExp { + if (dialect === 'mssql') { + if (!mssqlIntegerLiteralRE) { + mssqlIntegerLiteralRE = new RegExp('(? 0 ? truncated.substring(0, lastSpace) : truncated; } -let integerLiteralRE: RegExp | undefined; -let mssqlIntegerLiteralRE: RegExp | undefined; - -/** - * The lookbehind keeps `$n` and `?n` parameter placeholders out of the match. T-SQL has neither, - * and `$1000` there is a money literal, so `$` only guards a placeholder outside `mssql`. - * - * Lazy init: constructing a lookbehind at module scope evaluates it on import and crashes - * Safari <16.4 browser bundles that reach this file via the core barrel. - */ -function getIntegerLiteralRE(dialect: SqlDialect): RegExp { - if (dialect === 'mssql') { - if (!mssqlIntegerLiteralRE) { - mssqlIntegerLiteralRE = new RegExp('(?(); + + function getScopes(): TestStore { + return asyncStorage.getStore() || { scope: getDefaultCurrentScope(), isolationScope: getDefaultIsolationScope() }; + } + + setAsyncContextStrategy({ + withScope: callback => { + const scope = getScopes().scope.clone(); + const isolationScope = getScopes().isolationScope; + return asyncStorage.run({ scope, isolationScope }, () => callback(scope)); + }, + withSetScope: (scope, callback) => { + const isolationScope = getScopes().isolationScope; + return asyncStorage.run({ scope, isolationScope }, () => callback(scope)); + }, + withIsolationScope: callback => { + const scope = getScopes().scope; + const isolationScope = getScopes().isolationScope.clone(); + return asyncStorage.run({ scope, isolationScope }, () => callback(isolationScope)); + }, + withSetIsolationScope: (isolationScope, callback) => { + const scope = getScopes().scope; + return asyncStorage.run({ scope, isolationScope }, () => callback(isolationScope)); + }, + getCurrentScope: () => getScopes().scope, + getIsolationScope: () => getScopes().isolationScope, + getTracingChannelBinding: () => ({ + asyncLocalStorage: asyncStorage, + getStoreWithActiveSpan: span => { + const scope = getScopes().scope.clone(); + const isolationScope = getScopes().isolationScope; + _INTERNAL_setSpanForScope(scope, span); + return { scope, isolationScope }; + }, + }), + }); +} + +function makeSpan(): Span { + return { end: vi.fn(), setStatus: vi.fn(), setAttributes: vi.fn() } as unknown as Span; +} + +interface ChannelContext { + arguments: unknown[]; + self?: unknown; + result?: unknown; +} + +// `ClickHouseClient` as `>=1.10` builds it: the username sits in `auth`. +const CLIENT = { + connectionParams: { + url: new URL('http://clickhouse.internal:8123'), + database: 'analytics', + auth: { username: 'reporter', password: 'secret', type: 'Credentials' }, + }, +}; + +const CONNECTION_ATTRIBUTES = { + [SENTRY_OP]: DB, + [SENTRY_KIND]: 'client', + [SENTRY_ORIGIN]: 'auto.db.clickhouse', + [DB_SYSTEM_NAME]: 'clickhouse', + [DB_NAMESPACE]: 'analytics', + [DB_USER]: 'reporter', + [SERVER_ADDRESS]: 'clickhouse.internal', + [SERVER_PORT]: 8123, +}; + +describe('clickhouseIntegration', () => { + let startInactiveSpanSpy: MockInstance; + let captureExceptionSpy: MockInstance; + let span: Span; + + beforeAll(() => { + installTestAsyncContextStrategy(); + GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@clickhouse/client', '@clickhouse/client-common'] }; + clickhouseIntegration().setup?.({ on: () => () => undefined } as unknown as Client); + }); + + afterAll(() => { + setAsyncContextStrategy(undefined); + delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; + }); + + beforeEach(() => { + span = makeSpan(); + startInactiveSpanSpy = vi.spyOn(SentryCore, 'startInactiveSpan').mockReturnValue(span); + captureExceptionSpy = vi.spyOn(SentryCore, 'captureException'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + ['query', CHANNELS.CLICKHOUSE_QUERY], + ['command', CHANNELS.CLICKHOUSE_COMMAND], + ['exec', CHANNELS.CLICKHOUSE_EXEC], + ])('builds a db span for `%s` from the sanitized statement', async (_method, channel) => { + const ctx: ChannelContext = { + arguments: [{ query: "SELECT count() FROM events WHERE name = 'signup' AND id > 42" }], + self: CLIENT, + }; + + await tracingChannel(channel).tracePromise(async () => ({}), ctx); + + expect(startInactiveSpanSpy).toHaveBeenCalledWith({ + name: 'SELECT events', + onlyIfParent: true, + attributes: expect.objectContaining({ + ...CONNECTION_ATTRIBUTES, + [DB_QUERY_TEXT]: 'SELECT count() FROM events WHERE name = ? AND id > ?', + [DB_QUERY_SUMMARY]: 'SELECT events', + [DB_OPERATION_NAME]: 'SELECT', + }), + }); + expect(span.end).toHaveBeenCalledTimes(1); + }); + + it('builds a db span for `insert` from the target table', async () => { + const ctx: ChannelContext = { + arguments: [{ table: 'events', values: [{ id: 1, name: 'signup' }], format: 'JSONEachRow' }], + self: CLIENT, + }; + + await tracingChannel(CHANNELS.CLICKHOUSE_INSERT).tracePromise(async () => ({ executed: true }), ctx); + + expect(startInactiveSpanSpy).toHaveBeenCalledWith({ + name: 'INSERT events', + onlyIfParent: true, + attributes: expect.objectContaining({ + ...CONNECTION_ATTRIBUTES, + [DB_OPERATION_NAME]: 'INSERT', + [DB_COLLECTION_NAME]: 'events', + [DB_QUERY_SUMMARY]: 'INSERT events', + }), + }); + // The inserted values never reach the span. + expect(startInactiveSpanSpy.mock.calls[0]?.[0].attributes[DB_QUERY_TEXT]).toBeUndefined(); + expect(span.end).toHaveBeenCalledTimes(1); + }); + + it('skips `insert` with an empty array, which sends no request', async () => { + const ctx: ChannelContext = { arguments: [{ table: 'events', values: [] }], self: CLIENT }; + + await tracingChannel(CHANNELS.CLICKHOUSE_INSERT).tracePromise(async () => ({ executed: false }), ctx); + + expect(startInactiveSpanSpy).not.toHaveBeenCalled(); + }); + + it('sets an error status on a failed call without capturing the error', async () => { + const ctx: ChannelContext = { arguments: [{ query: 'SELECT * FROM does_not_exist' }], self: CLIENT }; + + await expect( + tracingChannel(CHANNELS.CLICKHOUSE_QUERY).tracePromise(async () => { + throw new Error('Table analytics.does_not_exist does not exist'); + }, ctx), + ).rejects.toThrow('Table analytics.does_not_exist does not exist'); + + expect(span.setStatus).toHaveBeenCalledWith({ + code: SPAN_STATUS_ERROR, + message: 'Table analytics.does_not_exist does not exist', + }); + expect(span.end).toHaveBeenCalledTimes(1); + expect(captureExceptionSpy).not.toHaveBeenCalled(); + }); + + it.each([ + ['query', CHANNELS.CLICKHOUSE_COMMON_QUERY], + ['command', CHANNELS.CLICKHOUSE_COMMON_COMMAND], + ['exec', CHANNELS.CLICKHOUSE_COMMON_EXEC], + ['insert', CHANNELS.CLICKHOUSE_COMMON_INSERT], + ])('builds a db span for `%s` from `@clickhouse/client-common`', async (_method, channel) => { + const ctx: ChannelContext = { arguments: [{ query: 'SELECT 1', table: 'events', values: [1] }], self: CLIENT }; + + await tracingChannel(channel).tracePromise(async () => ({}), ctx); + + expect(startInactiveSpanSpy).toHaveBeenCalledWith( + expect.objectContaining({ attributes: expect.objectContaining(CONNECTION_ATTRIBUTES) }), + ); + expect(span.end).toHaveBeenCalledTimes(1); + }); + + it('reads the username from older clients and the default port from the protocol', async () => { + const ctx: ChannelContext = { + arguments: [{ query: 'SELECT 1' }], + // `<1.10` keeps `username` at the top level of `connectionParams`. + self: { connectionParams: { url: new URL('https://ch.example.com'), database: 'default', username: 'default' } }, + }; + + await tracingChannel(CHANNELS.CLICKHOUSE_COMMON_QUERY).tracePromise(async () => ({}), ctx); + + expect(startInactiveSpanSpy).toHaveBeenCalledWith( + expect.objectContaining({ + attributes: expect.objectContaining({ + [DB_USER]: 'default', + [SERVER_ADDRESS]: 'ch.example.com', + [SERVER_PORT]: 443, + }), + }), + ); + }); + + it('prefers a per-call `auth` override for the username', async () => { + const ctx: ChannelContext = { + arguments: [{ query: 'SELECT 1', auth: { username: 'writer', password: 'secret' } }], + self: CLIENT, + }; + + await tracingChannel(CHANNELS.CLICKHOUSE_QUERY).tracePromise(async () => ({}), ctx); + + expect(startInactiveSpanSpy).toHaveBeenCalledWith( + expect.objectContaining({ attributes: expect.objectContaining({ [DB_USER]: 'writer' }) }), + ); + }); +}); diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index b04ce144b977..7ac5ef1b961a 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -769,6 +769,41 @@ describe('sanitizeSqlQuery', () => { }); }); + describe("dialect: 'clickhouse'", () => { + it.each([ + [String.raw`SELECT "a\"b", 'private@example.com' AS "c" FROM t`, String.raw`SELECT "a\"b", ? AS "c" FROM t`], + ["SELECT `a\\`b`, 'private@example.com' AS `c` FROM t", 'SELECT `a\\`b`, ? AS `c` FROM t'], + ['INSERT INTO events FORMAT CSV\n1,private@example.com', 'INSERT INTO events'], + [ + '/* comment */ INSERT INTO events (id, name) FORMAT JSONEachRow\n{"id":1,"name":"private@example.com"}', + 'INSERT INTO events (id, name)', + ], + ['INSERT INTO events FORMAT /* input */ CSV\n1,private@example.com', 'INSERT INTO events'], + ['INSERT INTO format (id) VALUES (1)', 'INSERT INTO format (id) VALUES (?)'], + ['INSERT INTO events SELECT format FROM source', 'INSERT INTO events SELECT format FROM source'], + ["INSERT INTO events (`format`) VALUES ('private@example.com')", 'INSERT INTO events (`format`) VALUES (?)'], + // `"..."` and backticks quote identifiers, as in `standard` + ['SELECT * FROM "events" WHERE `user` = \'jane@example.com\'', 'SELECT * FROM "events" WHERE `user` = ?'], + // `\` escapes the next character inside a string literal + [String.raw`SELECT * FROM t WHERE name = 'O\'Brien' AND b = 'x'`, 'SELECT * FROM t WHERE name = ? AND b = ?'], + [String.raw`SELECT * FROM t WHERE a = 'x\\' AND b = 'y'`, 'SELECT * FROM t WHERE a = ? AND b = ?'], + // `#` starts a line comment + ["SELECT * FROM t WHERE a = 'x' # trailing comment", 'SELECT * FROM t WHERE a = ?'], + // heredocs are dollar-quoted literals + ['SELECT $heredoc$secret$heredoc$ FROM t', 'SELECT ? FROM t'], + ['SELECT $1$private@example.com$1$ FROM t', 'SELECT ? FROM t'], + ['SELECT 10_000, 0b101010, 1_000.25, 0xDE_AD FROM events', 'SELECT ?, ?, ?, ? FROM events'], + ['SELECT .1_25, -10_000, 1_0e+2 FROM events', 'SELECT ?, ?, ? FROM events'], + // `{name:Type}` query parameters survive, including the digits in the type name + [ + 'SELECT * FROM t WHERE id = {id:UInt32} AND name = {name:String}', + 'SELECT * FROM t WHERE id = {id:UInt32} AND name = {name:String}', + ], + ])('sanitizes %p', (input, expected) => { + expect(sanitizeSqlQuery(input, 'clickhouse')).toBe(expected); + }); + }); + describe('unterminated literals swallow the rest of the statement', () => { it.each([ ["SELECT * FROM t WHERE a = 'jane@example.com AND b = 2", 'standard' as const], @@ -913,6 +948,7 @@ describe('toSqlDialect', () => { ['mssql', 'mssql'], ['sqlserver', 'mssql'], ['microsoft.sql_server', 'mssql'], + ['clickhouse', 'clickhouse'], ])('maps %j to %j', (system, expected) => { expect(toSqlDialect(system)).toBe(expected); }); diff --git a/packages/solidstart/src/server/index.ts b/packages/solidstart/src/server/index.ts index 39c5b31b1d38..302f60d45ce5 100644 --- a/packages/solidstart/src/server/index.ts +++ b/packages/solidstart/src/server/index.ts @@ -87,6 +87,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, redisIntegration, requestDataIntegration, diff --git a/packages/sveltekit/src/server/index.ts b/packages/sveltekit/src/server/index.ts index 77fcad4418d7..32eb93e5a05d 100644 --- a/packages/sveltekit/src/server/index.ts +++ b/packages/sveltekit/src/server/index.ts @@ -87,6 +87,7 @@ export { postgresIntegration, postgresJsIntegration, neonIntegration, + clickhouseIntegration, prismaIntegration, redisIntegration, requestDataIntegration, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92d07b80be6b..59df360d50ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -618,6 +618,9 @@ importers: '@aws-sdk/client-sqs': specifier: ^3.1161.0 version: 3.1161.0 + '@clickhouse/client': + specifier: 1.25.0 + version: 1.25.0 '@earendil-works/chord': specifier: ^1.4.0 version: 1.4.0 @@ -4153,6 +4156,10 @@ packages: resolution: {integrity: sha512-dlT1m5e/0yUL0kRNcQn7yGLVThkgbB0Ga/1AmfDDC/8ik6AIiSf2QLQO2zPYvefsHP0aFgxO93cVLCCfDp7kzQ==} engines: {node: '>= 20.12.0'} + '@clickhouse/client@1.25.0': + resolution: {integrity: sha512-sxoCDlA8iOEGpGrJzl9SeNvG7II6e5kSeo0JI4Mru2QhwhjydWuDEE55ryZg3vIJ0CSJ1bhn9+Xtnuvf1BZT/w==} + engines: {node: '>=20'} + '@cloudflare/kv-asset-handler@0.4.2': resolution: {integrity: sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ==} engines: {node: '>=18.0.0'} @@ -22009,6 +22016,8 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 + '@clickhouse/client@1.25.0': {} + '@cloudflare/kv-asset-handler@0.4.2': {} '@cloudflare/kv-asset-handler@0.5.0': {} From 770db7702c550b19028e854c3889114c387c6389 Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:09:53 -0600 Subject: [PATCH 2/8] fix(node): Strip ClickHouse FORMAT data after INSERT SELECT The sanitizer stopped looking for the FORMAT clause once an INSERT reached SELECT, so rows read through input() stayed in db.query.text. FORMAT is now the data clause anywhere at the top level of an INSERT, unless the token before it expects a name, as in SELECT format or FROM format. --- .../server-utils/src/utils/sql-sanitizer.ts | 94 +++++++++++++++---- packages/server-utils/test/utils/sql.test.ts | 20 ++++ 2 files changed, 94 insertions(+), 20 deletions(-) diff --git a/packages/server-utils/src/utils/sql-sanitizer.ts b/packages/server-utils/src/utils/sql-sanitizer.ts index 613a04b228bd..6dbd077bca7c 100644 --- a/packages/server-utils/src/utils/sql-sanitizer.ts +++ b/packages/server-utils/src/utils/sql-sanitizer.ts @@ -59,33 +59,83 @@ const CLICKHOUSE_DOLLAR_QUOTE_RE = /\$\w*\$/y; const CLICKHOUSE_NUMBER_RE = /-?(?:0x[\dA-F_]+|0b[01_]+|(?:\d[\d_]*(?:\.[\d_]*)?|\.\d[\d_]*)(?:e[+-]?[\d_]+)?)(?![\w$])/iy; +const CLICKHOUSE_WORD_RE = /[A-Za-z_][\w$]*/y; + +// A FORMAT clause follows a complete table name or expression. After one of these tokens `format` +// is a table, column or alias name instead. +const CLICKHOUSE_OPERAND_PREFIXES = new Set([ + 'INTO', + 'TABLE', + 'SELECT', + 'DISTINCT', + 'FROM', + 'JOIN', + 'USING', + 'ON', + 'WHERE', + 'PREWHERE', + 'HAVING', + 'BY', + 'AND', + 'OR', + 'NOT', + 'AS', + 'CASE', + 'WHEN', + 'THEN', + 'ELSE', + 'IN', + 'IS', + 'LIKE', + 'ILIKE', + 'BETWEEN', + ',', + '.', + '=', + '<', + '>', + '!', + '+', + '-', + '/', + '%', + '|', + '&', +]); + interface ClickhouseInsertState { - insert: boolean; - lastKeyword?: string; + /** Whether the statement is an INSERT, decided by its first word. */ + insert?: boolean; + /** The last token: an upper-cased word, a punctuation character, or `''` for a literal or quoted identifier. */ + previous?: string; depth: number; } -function isClickhouseInsertData(sql: string, start: number, state: ClickhouseInsertState): boolean { - const char = sql[start]!; +/** + * Whether `char` starts the inline data of an INSERT. Everything after the top-level FORMAT clause is + * input data, which cannot be sanitized as SQL. That includes `INSERT ... SELECT`, where the + * `input()` table function reads that data. + */ +function isClickhouseInsertData(sql: string, start: number, char: string, state: ClickhouseInsertState): boolean { if (char === '(') state.depth++; if (char === ')') state.depth--; - if (!/[A-Za-z_]/.test(char) || isIdentifierChar(sql[start - 1])) { + if (/\s/.test(char) || (isIdentifierChar(char) && isIdentifierChar(sql[start - 1]))) { return false; } - const keyword = /^[A-Za-z_][\w$]*/.exec(sql.slice(start))![0].toUpperCase(); - if (state.lastKeyword === undefined) { - state.insert = keyword === 'INSERT'; + CLICKHOUSE_WORD_RE.lastIndex = start; + const word = CLICKHOUSE_WORD_RE.exec(sql)?.[0].toUpperCase(); + if (word) { + state.insert ??= word === 'INSERT'; } - if (state.insert && state.depth === 0 && state.lastKeyword !== 'INTO' && sql[start - 1] !== '.') { - // FORMAT starts arbitrary input data, which cannot be sanitized as SQL. - if (keyword === 'FORMAT') { - return true; - } - if (keyword === 'SELECT') { - state.insert = false; - } + if ( + state.insert && + state.depth === 0 && + word === 'FORMAT' && + !CLICKHOUSE_OPERAND_PREFIXES.has(state.previous ?? '') + ) { + return true; } - state.lastKeyword = keyword; + state.previous = word ?? char; return false; } @@ -136,7 +186,8 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { const isClickhouse = dialect === 'clickhouse'; const out: string[] = []; let i = 0; - const clickhouseInsert: ClickhouseInsertState = { insert: false, depth: 0 }; + // A literal, number or quoted identifier completes an operand, so it resets `previous`. + const clickhouseInsert: ClickhouseInsertState = { depth: 0 }; while (i < sql.length) { const char = sql[i]!; @@ -165,6 +216,7 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { const bodyEnd = sql.indexOf(tag, i + tag.length); i = bodyEnd === -1 ? sql.length : bodyEnd + tag.length; out.push('?'); + clickhouseInsert.previous = ''; continue; } } @@ -176,7 +228,7 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { // Copying would carry every literal in the rest of the statement through unlexed. out.push(runEnd === -1 ? '?' : sql.slice(i, runEnd)); i = runEnd === -1 ? sql.length : runEnd; - clickhouseInsert.lastKeyword = ''; + clickhouseInsert.previous = ''; continue; } @@ -190,6 +242,7 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { const runEnd = findQuotedRunEnd(sql, i, char, isMysql || isClickhouse || prefix === 'E'); i = runEnd === -1 ? sql.length : runEnd; out.push('?'); + clickhouseInsert.previous = ''; continue; } @@ -197,9 +250,10 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { if (number) { out.push('?'); i += number.length; + clickhouseInsert.previous = ''; continue; } - if (isClickhouse && isClickhouseInsertData(sql, i, clickhouseInsert)) { + if (isClickhouse && isClickhouseInsertData(sql, i, char, clickhouseInsert)) { break; } out.push(char); diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index 7ac5ef1b961a..a331fa839adf 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -781,6 +781,26 @@ describe('sanitizeSqlQuery', () => { ['INSERT INTO events FORMAT /* input */ CSV\n1,private@example.com', 'INSERT INTO events'], ['INSERT INTO format (id) VALUES (1)', 'INSERT INTO format (id) VALUES (?)'], ['INSERT INTO events SELECT format FROM source', 'INSERT INTO events SELECT format FROM source'], + // `input()` reads the rows that follow the `FORMAT` clause of an `INSERT ... SELECT` + [ + "INSERT INTO events SELECT * FROM input('email String') FORMAT CSV\nprivate@example.com", + 'INSERT INTO events SELECT * FROM input(?)', + ], + [ + 'INSERT INTO events SELECT lower(email) FROM input(\'email String\') WHERE email != \'\' FORMAT JSONEachRow\n{"email":"private@example.com"}', + 'INSERT INTO events SELECT lower(email) FROM input(?) WHERE email != ?', + ], + [ + "INSERT INTO events SELECT * FROM input('n UInt8') LIMIT 10 FORMAT TSV\nprivate@example.com", + 'INSERT INTO events SELECT * FROM input(?) LIMIT ?', + ], + ['INSERT INTO events SELECT id, format FROM source', 'INSERT INTO events SELECT id, format FROM source'], + ['INSERT INTO events SELECT * FROM format', 'INSERT INTO events SELECT * FROM format'], + [ + 'INSERT INTO events SELECT * FROM source WHERE format = 1 ORDER BY format', + 'INSERT INTO events SELECT * FROM source WHERE format = ? ORDER BY format', + ], + ['INSERT INTO TABLE format FORMAT CSV\nprivate@example.com', 'INSERT INTO TABLE format'], ["INSERT INTO events (`format`) VALUES ('private@example.com')", 'INSERT INTO events (`format`) VALUES (?)'], // `"..."` and backticks quote identifiers, as in `standard` ['SELECT * FROM "events" WHERE `user` = \'jane@example.com\'', 'SELECT * FROM "events" WHERE `user` = ?'], From e9b9f371d84d2addd646c5a27b4896935b9c56c4 Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:04:01 -0600 Subject: [PATCH 3/8] fix(node): Strip nested ClickHouse comments --- .../suites/tracing/clickhouse/scenario.mjs | 6 ++++++ .../suites/tracing/clickhouse/test.ts | 2 ++ .../server-utils/src/utils/sql-sanitizer.ts | 21 +++++++++++++++++-- packages/server-utils/test/utils/sql.test.ts | 16 ++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs index 01fc63ba13ae..5417bfeb9fc0 100644 --- a/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs @@ -33,6 +33,12 @@ async function run() { }); await literal.json(); + const nestedComment = await client.query({ + query: 'SELECT 1 /* outer /* inner */ private@example.com */', + format: 'JSONEachRow', + }); + await nestedComment.json(); + const parameterized = await client.query({ query: 'SELECT name FROM events WHERE id = {id:UInt32}', query_params: { id: 1 }, diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts index d91dcf29bf39..7e221384bc93 100644 --- a/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts @@ -40,6 +40,8 @@ const EXPECTED_SPANS = [ { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: 'INSERT INTO events' }, // the string literal is sanitized out of the statement { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT name FROM events WHERE name = ?' }, + // nested comments are removed before the query reaches the span + { name: 'SELECT', status: 'ok', operation: 'SELECT', text: 'SELECT ?' }, // `{name:Type}` query parameters are kept as written { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT name FROM events WHERE id = {id:UInt32}' }, { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT count() FROM events FORMAT JSONEachRow' }, diff --git a/packages/server-utils/src/utils/sql-sanitizer.ts b/packages/server-utils/src/utils/sql-sanitizer.ts index 6dbd077bca7c..e38d8d0088f1 100644 --- a/packages/server-utils/src/utils/sql-sanitizer.ts +++ b/packages/server-utils/src/utils/sql-sanitizer.ts @@ -170,6 +170,24 @@ function findQuotedRunEnd(sql: string, start: number, delimiter: string, backsla return -1; } +// An unterminated comment consumes the rest of the query. +function findBlockCommentEnd(sql: string, start: number, nested: boolean): number { + let depth = 1; + let i = start + 2; + while (i < sql.length && depth > 0) { + if (sql[i] === '*' && sql[i + 1] === '/') { + depth--; + i += 2; + } else if (nested && sql[i] === '/' && sql[i + 1] === '*') { + depth++; + i += 2; + } else { + i++; + } + } + return i; +} + /** * Replaces every string literal with `?` and drops every comment, in one pass. * @@ -200,8 +218,7 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { } if (char === '/' && next === '*') { - const commentEnd = sql.indexOf('*/', i + 2); - i = commentEnd === -1 ? sql.length : commentEnd + 2; + i = findBlockCommentEnd(sql, i, isClickhouse); continue; } diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index a331fa839adf..1af8376c0fdd 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -770,6 +770,22 @@ describe('sanitizeSqlQuery', () => { }); describe("dialect: 'clickhouse'", () => { + it.each([ + ['SELECT 1 /* outer /* inner */ private@example.com */', 'SELECT ?'], + ['SELECT /* outer /* inner */ private@example.com */ 1', 'SELECT ?'], + ['SELECT 1 /* outer /* middle /* inner */ private@example.com */ outer */ + 2', 'SELECT ? + ?'], + ["SELECT 1 /* outer ' /* inner */ private@example.com */ + 2", 'SELECT ? + ?'], + ['SELECT 1 /* outer /* inner */ private@example.com', 'SELECT ?'], + ['SELECT 1 /* outer /* inner private@example.com', 'SELECT ?'], + ['SELECT 1 /* outer /* inner */ private@example.com */ /* other /* inner */ secret */', 'SELECT ?'], + [ + 'INSERT INTO events /* outer /* inner */ private@example.com */ FORMAT CSV\n1,private@example.com', + 'INSERT INTO events', + ], + ])('removes the entire nested comment in %p', (input, expected) => { + expect(sanitizeSqlQuery(input, 'clickhouse')).toBe(expected); + }); + it.each([ [String.raw`SELECT "a\"b", 'private@example.com' AS "c" FROM t`, String.raw`SELECT "a\"b", ? AS "c" FROM t`], ["SELECT `a\\`b`, 'private@example.com' AS `c` FROM t", 'SELECT `a\\`b`, ? AS `c` FROM t'], From 4f8327341d946091d0434b78c7986b33f13bbf28 Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:54:30 -0600 Subject: [PATCH 4/8] test(bun): Run the ClickHouse suite only with bundled scenarios `bun run` does not inject the diagnostics channels, so the ClickHouse suite gets no spans there. It passes in the `node-suites-bun-build` project, which bundles the scenarios with `@sentry/bun/plugin`. --- dev-packages/bun-integration-tests/node-suites/excludes.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index 6d775876f475..2d6a627d8d87 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -81,6 +81,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/anthropic/v0.129/test.ts', 'suites/tracing/anthropic/v1/test.ts', 'suites/tracing/apollo-graphql/**', + 'suites/tracing/clickhouse/test.ts', 'suites/tracing/dataloader/test.ts', 'suites/tracing/fastify/test.ts', 'suites/tracing/genericPool-v2/test.ts', From 105057442fde52cc3ca8d37eeee256bf9915715f Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:04:18 -0600 Subject: [PATCH 5/8] fix(node): Address ClickHouse review feedback --- .../cloudflare-integration-tests/package.json | 1 + .../suites/tracing/clickhouse/index.ts | 58 +++ .../tracing/clickhouse/instrument.server.ts | 10 + .../suites/tracing/clickhouse/test.ts | 111 ++++++ .../suites/tracing/clickhouse/vite.config.mts | 7 + .../suites/tracing/clickhouse/wrangler.jsonc | 7 + .../node-integration-tests/package.json | 2 +- .../suites/tracing/clickhouse/scenario.mjs | 1 + .../suites/tracing/clickhouse/test.ts | 1 + .../src/integrations/clickhouse.ts | 19 +- .../config/channel-integration-definitions.ts | 5 +- .../src/orchestrion/config/clickhouse.ts | 8 +- .../server-utils/src/utils/sql-sanitizer.ts | 367 ----------------- packages/server-utils/src/utils/sql.ts | 376 +++++++++++++++++- .../test/integrations/clickhouse.test.ts | 103 +++-- packages/server-utils/test/utils/sql.test.ts | 4 + pnpm-lock.yaml | 10 +- 17 files changed, 667 insertions(+), 423 deletions(-) create mode 100644 dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/index.ts create mode 100644 dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/instrument.server.ts create mode 100644 dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts create mode 100644 dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/vite.config.mts create mode 100644 dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/wrangler.jsonc delete mode 100644 packages/server-utils/src/utils/sql-sanitizer.ts diff --git a/dev-packages/cloudflare-integration-tests/package.json b/dev-packages/cloudflare-integration-tests/package.json index 9fce2e48b148..593fc024aa90 100644 --- a/dev-packages/cloudflare-integration-tests/package.json +++ b/dev-packages/cloudflare-integration-tests/package.json @@ -15,6 +15,7 @@ }, "devDependencies": { "@anthropic-ai/sdk": "0.133.0", + "@clickhouse/client-web": "^1.25.0", "@cloudflare/vite-plugin": "1.66.0", "@cloudflare/workers-types": "^4.20260426.0", "@cloudflare/workers-types-v5": "npm:@cloudflare/workers-types@5.20260710.1", diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/index.ts new file mode 100644 index 000000000000..e9143ff84d6c --- /dev/null +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/index.ts @@ -0,0 +1,58 @@ +import { createClient } from '@clickhouse/client-web'; + +// The real web client runs in workerd, with its methods instrumented by the Sentry Vite plugin. +// Only the HTTP responses are mocked, so this needs neither a ClickHouse service nor credentials. +export default { + async fetch() { + const queries: string[] = []; + const mockFetch: typeof fetch = async (input, init) => { + const url = new URL(String(input)); + const query = url.searchParams.get('query') ?? String(init?.body); + queries.push(query); + if (query.includes('does_not_exist')) { + return new Response('Code: 60. DB::Exception: Table tests.does_not_exist does not exist. (UNKNOWN_TABLE)', { + status: 500, + }); + } + return new Response('{"name":"signup"}\n'); + }; + + const client = createClient({ + url: 'http://clickhouse.internal:8123', + database: 'tests', + username: 'test', + password: 'test', + fetch: mockFetch, + }); + + try { + const result = await client.query({ + query: "SELECT name FROM events WHERE name = 'signup'", + format: 'JSONEachRow', + }); + const rows = await result.json(); + await client.command({ query: 'INSERT INTO TABLE events FORMAT CSV\n1,private@example.com' }); + const { stream } = await client.exec({ query: 'SELECT count() FROM events FORMAT JSONEachRow' }); + await stream.cancel(); + await client.insert({ table: 'events', values: [{ id: 1, name: 'signup' }], format: 'JSONEachRow' }); + // An empty insert makes no HTTP request and must not create a span. + await client.insert({ table: 'events', values: [], format: 'JSONEachRow' }); + + let failures = 0; + for (const call of [ + () => client.query({ query: 'SELECT * FROM does_not_exist' }), + () => client.command({ query: 'DROP TABLE does_not_exist' }), + () => client.exec({ query: 'SELECT * FROM does_not_exist FORMAT JSONEachRow' }), + () => client.insert({ table: 'does_not_exist', values: [{ id: 1 }], format: 'JSONEachRow' }), + ]) { + await call().catch(() => { + failures++; + }); + } + + return Response.json({ rows, requests: queries.length, failures }); + } finally { + await client.close(); + } + }, +} satisfies ExportedHandler; diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/instrument.server.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/instrument.server.ts new file mode 100644 index 000000000000..2659d1a477a6 --- /dev/null +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/instrument.server.ts @@ -0,0 +1,10 @@ +import { defineCloudflareOptions } from '@sentry/cloudflare'; + +interface Env { + SENTRY_DSN: string; +} + +export default defineCloudflareOptions((env: Env) => ({ + dsn: env.SENTRY_DSN, + tracesSampleRate: 1.0, +})); diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts new file mode 100644 index 000000000000..1faf6cb17959 --- /dev/null +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts @@ -0,0 +1,111 @@ +import { + DB_COLLECTION_NAME, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { expect, it } from 'vitest'; +import { createRunner } from '../../../runner'; +import { getSpansFromEnvelope } from '../../../spanUtils'; + +it('instruments client-web calls in a bundled Worker and leaves handled errors as spans', async ({ signal }) => { + const runner = createRunner(__dirname).start(signal); + const spansPromise = runner.collectStreamedSpans( + spans => + spans.some(span => span.is_segment) && + spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.db.clickhouse').length === 8, + ); + + expect(await runner.makeRequest('get', '/')).toEqual({ rows: [{ name: 'signup' }], requests: 8, failures: 4 }); + const spans = await spansPromise; + const segment = spans.find(span => span.is_segment); + const dbSpans = spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.db.clickhouse'); + + for (const span of dbSpans) { + expect(span.parent_span_id).toBe(segment?.span_id); + expect(span.trace_id).toBe(segment?.trace_id); + expect(span.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'db' }); + expect(span.attributes[SENTRY_KIND]).toEqual({ type: 'string', value: 'client' }); + expect(span.attributes[DB_SYSTEM_NAME]).toEqual({ type: 'string', value: 'clickhouse' }); + expect(span.attributes[DB_NAMESPACE]).toEqual({ type: 'string', value: 'tests' }); + expect(span.attributes[DB_USER]).toEqual({ type: 'string', value: 'test' }); + expect(span.attributes[SERVER_ADDRESS]).toEqual({ type: 'string', value: 'clickhouse.internal' }); + expect(span.attributes[SERVER_PORT]).toEqual({ type: 'integer', value: 8123 }); + expect(span.attributes[DB_QUERY_SUMMARY]?.value).toBe(span.name); + } + + expect( + dbSpans.map(span => ({ + name: span.name, + status: span.status, + operation: span.attributes[DB_OPERATION_NAME]?.value, + text: span.attributes[DB_QUERY_TEXT]?.value, + collection: span.attributes[DB_COLLECTION_NAME]?.value, + })), + ).toEqual([ + { + name: 'SELECT events', + status: 'ok', + operation: 'SELECT', + text: 'SELECT name FROM events WHERE name = ?', + collection: undefined, + }, + { + name: 'INSERT events', + status: 'ok', + operation: 'INSERT', + text: 'INSERT INTO TABLE events', + collection: undefined, + }, + { + name: 'SELECT events', + status: 'ok', + operation: 'SELECT', + text: 'SELECT count() FROM events FORMAT JSONEachRow', + collection: undefined, + }, + { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: undefined, collection: 'events' }, + { + name: 'SELECT does_not_exist', + status: 'error', + operation: 'SELECT', + text: 'SELECT * FROM does_not_exist', + collection: undefined, + }, + { + name: 'DROP TABLE does_not_exist', + status: 'error', + operation: 'DROP', + text: 'DROP TABLE does_not_exist', + collection: undefined, + }, + { + name: 'SELECT does_not_exist', + status: 'error', + operation: 'SELECT', + text: 'SELECT * FROM does_not_exist FORMAT JSONEachRow', + collection: undefined, + }, + { + name: 'INSERT does_not_exist', + status: 'error', + operation: 'INSERT', + text: undefined, + collection: 'does_not_exist', + }, + ]); + + const envelopes = runner.getReceivedEnvelopes(); + expect(envelopes.some(envelope => envelope[1].some(item => item[0].type === 'event'))).toBe(false); + expect( + envelopes.flatMap(getSpansFromEnvelope).filter(span => span.attributes[SENTRY_OP]?.value === 'db'), + ).toHaveLength(8); +}); diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/vite.config.mts b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/vite.config.mts new file mode 100644 index 000000000000..005f4448f6cb --- /dev/null +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/vite.config.mts @@ -0,0 +1,7 @@ +import { cloudflare } from '@cloudflare/vite-plugin'; +import { sentryCloudflareVitePlugin } from '@sentry/cloudflare/vite'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [cloudflare(), sentryCloudflareVitePlugin()], +}); diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/wrangler.jsonc b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/wrangler.jsonc new file mode 100644 index 000000000000..554a862db1f1 --- /dev/null +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/wrangler.jsonc @@ -0,0 +1,7 @@ +{ + "$schema": "../../../node_modules/wrangler/config-schema.json", + "name": "cloudflare-clickhouse", + "main": "index.ts", + "compatibility_date": "2025-06-17", + "compatibility_flags": ["nodejs_compat"], +} diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index ad59e88d7be7..0a3a70f7e6f4 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -35,7 +35,7 @@ "@aws-sdk/client-sfn": "^3.1161.0", "@aws-sdk/client-sns": "^3.1161.0", "@aws-sdk/client-sqs": "^3.1161.0", - "@clickhouse/client": "1.25.0", + "@clickhouse/client": "^1.25.0", "@earendil-works/chord": "^1.4.0", "@earendil-works/pi-ai": "^1.4.0", "@earendil-works/pi-durable": "^1.4.0", diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs index 5417bfeb9fc0..1540475e1e68 100644 --- a/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/scenario.mjs @@ -26,6 +26,7 @@ async function run() { await client.insert({ table: 'events', values: [{ id: 1, name: 'signup' }], format: 'JSONEachRow' }); await client.command({ query: 'INSERT INTO events FORMAT CSV\n2,inline-sensitive@example.com' }); + await client.command({ query: 'INSERT INTO TABLE events FORMAT CSV\n3,inline-sensitive@example.com' }); const literal = await client.query({ query: "SELECT name FROM events WHERE name = 'signup'", diff --git a/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts index 7e221384bc93..9db806a55a53 100644 --- a/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/clickhouse/test.ts @@ -38,6 +38,7 @@ const EXPECTED_SPANS = [ { name: 'CREATE TABLE events', status: 'ok', operation: 'CREATE', text: CREATE_TABLE }, { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: undefined, collection: 'events' }, { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: 'INSERT INTO events' }, + { name: 'INSERT events', status: 'ok', operation: 'INSERT', text: 'INSERT INTO TABLE events' }, // the string literal is sanitized out of the statement { name: 'SELECT events', status: 'ok', operation: 'SELECT', text: 'SELECT name FROM events WHERE name = ?' }, // nested comments are removed before the query reaches the span diff --git a/packages/server-utils/src/integrations/clickhouse.ts b/packages/server-utils/src/integrations/clickhouse.ts index bb3e5232caff..78e50c04da5e 100644 --- a/packages/server-utils/src/integrations/clickhouse.ts +++ b/packages/server-utils/src/integrations/clickhouse.ts @@ -66,13 +66,17 @@ const _clickhouseIntegration = (() => { }) satisfies IntegrationFn; function instrumentClickhouse(): void { - // `@clickhouse/client-common` (`<1.23.0`) and `@clickhouse/client` (`>=1.23.0`) publish the same - // calls under their own module names. + // The Node and web clients publish the same calls under their own module names. Before 1.23.0, + // both use `@clickhouse/client-common`. const channels: [string, ClickHouseMethod][] = [ [CHANNELS.CLICKHOUSE_QUERY, 'query'], [CHANNELS.CLICKHOUSE_COMMAND, 'command'], [CHANNELS.CLICKHOUSE_EXEC, 'exec'], [CHANNELS.CLICKHOUSE_INSERT, 'insert'], + [CHANNELS.CLICKHOUSE_WEB_QUERY, 'query'], + [CHANNELS.CLICKHOUSE_WEB_COMMAND, 'command'], + [CHANNELS.CLICKHOUSE_WEB_EXEC, 'exec'], + [CHANNELS.CLICKHOUSE_WEB_INSERT, 'insert'], [CHANNELS.CLICKHOUSE_COMMON_QUERY, 'query'], [CHANNELS.CLICKHOUSE_COMMON_COMMAND, 'command'], [CHANNELS.CLICKHOUSE_COMMON_EXEC, 'exec'], @@ -94,7 +98,9 @@ function instrumentClickhouse(): void { [SENTRY_KIND]: 'client', [SENTRY_ORIGIN]: ORIGIN, ...getConnectionAttributes(data.self?.connectionParams, params), - ...(method === 'insert' ? getInsertAttributes(params) : getStatementAttributes(params)), + ...(method === 'insert' + ? getInsertAttributes(params) + : getSqlQueryAttributes(typeof params?.query === 'string' ? params.query : undefined, 'clickhouse')), }; return startInactiveSpan({ @@ -106,11 +112,6 @@ function instrumentClickhouse(): void { } } -function getStatementAttributes(params: ClickHouseParams | undefined): SpanAttributes { - const query = params?.query; - return { ...getSqlQueryAttributes(typeof query === 'string' ? query : undefined, 'clickhouse') }; -} - // `insert` builds its `INSERT INTO
FORMAT ` statement internally, so the span is // described from the target table instead of a statement. function getInsertAttributes(params: ClickHouseParams | undefined): SpanAttributes { @@ -146,7 +147,7 @@ function getServerPort(url: URL): number { } /** - * Diagnostics-channel-based `@clickhouse/client` integration. + * Diagnostics-channel-based `@clickhouse/client` and `@clickhouse/client-web` integration. * * Subscribes to the channels Sentry's code transform injects into `ClickHouseClient`'s `query`, * `command`, `exec` and `insert` methods, and emits a `db` span for each call. Requires the Sentry diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 353f924e49d9..d8d81b3ab3ce 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -19,7 +19,10 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'postgresIntegration', modules: ['pg', 'pg-pool'] }, { exportName: 'postgresJsIntegration', modules: ['postgres'] }, { exportName: 'neonIntegration', modules: ['@neondatabase/serverless'] }, - { exportName: 'clickhouseIntegration', modules: ['@clickhouse/client', '@clickhouse/client-common'] }, + { + exportName: 'clickhouseIntegration', + modules: ['@clickhouse/client', '@clickhouse/client-web', '@clickhouse/client-common'], + }, { exportName: 'prismaIntegration', modules: ['@prisma/orm-family-sql'] }, { exportName: 'mysqlIntegration', modules: ['mysql'] }, { exportName: 'mysql2Integration', modules: ['mysql2'] }, diff --git a/packages/server-utils/src/orchestrion/config/clickhouse.ts b/packages/server-utils/src/orchestrion/config/clickhouse.ts index fe0953f914f1..b384748c4b70 100644 --- a/packages/server-utils/src/orchestrion/config/clickhouse.ts +++ b/packages/server-utils/src/orchestrion/config/clickhouse.ts @@ -8,7 +8,8 @@ import { getModuleNames } from './module-names'; // // Two file layouts across the supported range: // - `< 1.23.0` → the class lives in the separate `@clickhouse/client-common` package (`dist/client.js`) -// - `>= 1.23.0` → `@clickhouse/client` ships that code itself, in `dist/common/client.js` +// - `>= 1.23.0` → `@clickhouse/client` and `@clickhouse/client-web` each ship that code in +// `dist/common/client.js` // // Starts at 1.1.0: before it, `command` delegated to `exec` (two spans per command) and the client // did not keep the `connectionParams` the span attributes are read from. @@ -20,6 +21,7 @@ const METHODS = ['query', 'command', 'exec', 'insert'] as const; const MODULES = [ { name: '@clickhouse/client-common', versionRange: '>=1.1.0 <1.23.0', filePath: 'dist/client.js' }, { name: '@clickhouse/client', versionRange: '>=1.23.0 <2', filePath: 'dist/common/client.js' }, + { name: '@clickhouse/client-web', versionRange: '>=1.23.0 <2', filePath: 'dist/common/client.js' }, ]; export const clickhouseConfig: InstrumentationConfig[] = MODULES.flatMap(module => @@ -37,6 +39,10 @@ export const clickhouseChannels = { CLICKHOUSE_COMMAND: 'orchestrion:@clickhouse/client:command', CLICKHOUSE_EXEC: 'orchestrion:@clickhouse/client:exec', CLICKHOUSE_INSERT: 'orchestrion:@clickhouse/client:insert', + CLICKHOUSE_WEB_QUERY: 'orchestrion:@clickhouse/client-web:query', + CLICKHOUSE_WEB_COMMAND: 'orchestrion:@clickhouse/client-web:command', + CLICKHOUSE_WEB_EXEC: 'orchestrion:@clickhouse/client-web:exec', + CLICKHOUSE_WEB_INSERT: 'orchestrion:@clickhouse/client-web:insert', CLICKHOUSE_COMMON_QUERY: 'orchestrion:@clickhouse/client-common:query', CLICKHOUSE_COMMON_COMMAND: 'orchestrion:@clickhouse/client-common:command', CLICKHOUSE_COMMON_EXEC: 'orchestrion:@clickhouse/client-common:exec', diff --git a/packages/server-utils/src/utils/sql-sanitizer.ts b/packages/server-utils/src/utils/sql-sanitizer.ts deleted file mode 100644 index e38d8d0088f1..000000000000 --- a/packages/server-utils/src/utils/sql-sanitizer.ts +++ /dev/null @@ -1,367 +0,0 @@ -let integerLiteralRE: RegExp | undefined; -let mssqlIntegerLiteralRE: RegExp | undefined; - -/** - * The lookbehind keeps `$n` and `?n` parameter placeholders out of the match. T-SQL has neither, - * and `$1000` there is a money literal, so `$` only guards a placeholder outside `mssql`. - * - * Lazy init: constructing a lookbehind at module scope evaluates it on import and crashes - * Safari <16.4 browser bundles that reach this file via the core barrel. - */ -function getIntegerLiteralRE(dialect: SqlDialect): RegExp { - if (dialect === 'mssql') { - if (!mssqlIntegerLiteralRE) { - mssqlIntegerLiteralRE = new RegExp('(?', - '!', - '+', - '-', - '/', - '%', - '|', - '&', -]); - -interface ClickhouseInsertState { - /** Whether the statement is an INSERT, decided by its first word. */ - insert?: boolean; - /** The last token: an upper-cased word, a punctuation character, or `''` for a literal or quoted identifier. */ - previous?: string; - depth: number; -} - -/** - * Whether `char` starts the inline data of an INSERT. Everything after the top-level FORMAT clause is - * input data, which cannot be sanitized as SQL. That includes `INSERT ... SELECT`, where the - * `input()` table function reads that data. - */ -function isClickhouseInsertData(sql: string, start: number, char: string, state: ClickhouseInsertState): boolean { - if (char === '(') state.depth++; - if (char === ')') state.depth--; - if (/\s/.test(char) || (isIdentifierChar(char) && isIdentifierChar(sql[start - 1]))) { - return false; - } - CLICKHOUSE_WORD_RE.lastIndex = start; - const word = CLICKHOUSE_WORD_RE.exec(sql)?.[0].toUpperCase(); - if (word) { - state.insert ??= word === 'INSERT'; - } - if ( - state.insert && - state.depth === 0 && - word === 'FORMAT' && - !CLICKHOUSE_OPERAND_PREFIXES.has(state.previous ?? '') - ) { - return true; - } - state.previous = word ?? char; - return false; -} - -function matchClickhouseNumber(sql: string, start: number): string | undefined { - if (isIdentifierChar(sql[start - 1]) || sql[start - 1] === '?') { - return undefined; - } - CLICKHOUSE_NUMBER_RE.lastIndex = start; - return CLICKHOUSE_NUMBER_RE.exec(sql)?.[0]; -} - -/** - * Returns the index just past the run's closing `delimiter`, or -1 if the run never closes. The - * caller cannot derive that from the index alone: a query ending in `[a]]` consumed a doubled - * delimiter, not a closing one. - * - * A doubled delimiter (`''`) escapes itself in every dialect; backslash escapes are dialect- and - * context-dependent, so the caller decides. - */ -function findQuotedRunEnd(sql: string, start: number, delimiter: string, backslashEscapes: boolean): number { - for (let i = start + 1; i < sql.length; i++) { - const char = sql[i]; - if (backslashEscapes && char === '\\') { - i++; - } else if (char === delimiter) { - if (sql[i + 1] !== delimiter) { - return i + 1; - } - i++; - } - } - return -1; -} - -// An unterminated comment consumes the rest of the query. -function findBlockCommentEnd(sql: string, start: number, nested: boolean): number { - let depth = 1; - let i = start + 2; - while (i < sql.length && depth > 0) { - if (sql[i] === '*' && sql[i + 1] === '/') { - depth--; - i += 2; - } else if (nested && sql[i] === '/' && sql[i + 1] === '*') { - depth++; - i += 2; - } else { - i++; - } - } - return i; -} - -/** - * Replaces every string literal with `?` and drops every comment, in one pass. - * - * Doing this by scanning rather than by regex is what keeps quote state and comment state from - * being decided independently: a regex for `'...'` cannot see that the quote it stopped at was - * backslash-escaped, and a regex for `--...` cannot see that the `--` sits inside a literal. Both - * mistakes end with user data surviving into `db.query.text` and `db.query.summary`. - * - * Quoted identifiers are preserved — they are the table and column names the query summary is - * built from. - */ -function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { - const isMysql = dialect === 'mysql'; - const isClickhouse = dialect === 'clickhouse'; - const out: string[] = []; - let i = 0; - // A literal, number or quoted identifier completes an operand, so it resets `previous`. - const clickhouseInsert: ClickhouseInsertState = { depth: 0 }; - - while (i < sql.length) { - const char = sql[i]!; - const next = sql[i + 1]; - - if ((char === '-' && next === '-') || ((isMysql || isClickhouse) && char === '#')) { - const lineEnd = sql.indexOf('\n', i); - i = lineEnd === -1 ? sql.length : lineEnd; - continue; - } - - if (char === '/' && next === '*') { - i = findBlockCommentEnd(sql, i, isClickhouse); - continue; - } - - // In a dollar-quoted body (`$$body$$`, `$tag$body$tag$`) nothing has syntax meaning. Read the - // previous character from the query, not from `out`, where a dropped comment would leave `$$` - // looking like part of an identifier. Of the `standard` engines only PostgreSQL has dollar - // quoting. SQLite `$name` parameters do not match the tag pattern, apart from the rare `$a$b`. - // ClickHouse heredocs use the same syntax. - if ((dialect === 'standard' || isClickhouse) && char === '$' && !isIdentifierChar(sql[i - 1])) { - const tag = matchDollarQuoteTag(sql, i, isClickhouse); - if (tag) { - const bodyEnd = sql.indexOf(tag, i + tag.length); - i = bodyEnd === -1 ? sql.length : bodyEnd + tag.length; - out.push('?'); - clickhouseInsert.previous = ''; - continue; - } - } - - const identifierCloser = getIdentifierCloser(char, dialect); - if (identifierCloser) { - const runEnd = findQuotedRunEnd(sql, i, identifierCloser, isClickhouse); - // A run that never closes is not an identifier, so it collapses instead of being copied out. - // Copying would carry every literal in the rest of the statement through unlexed. - out.push(runEnd === -1 ? '?' : sql.slice(i, runEnd)); - i = runEnd === -1 ? sql.length : runEnd; - clickhouseInsert.previous = ''; - continue; - } - - if (char === "'" || (char === '"' && isMysql)) { - // A prefix like `X'1A'`, `B'01'`, `N'…'` or PostgreSQL's `E'a\nb'` is part of the literal, so it has - // to collapse into the same `?` instead of being left behind as a bare identifier. - const prefix = char === "'" ? getLiteralPrefix(out, dialect) : undefined; - if (prefix) { - out.pop(); - } - const runEnd = findQuotedRunEnd(sql, i, char, isMysql || isClickhouse || prefix === 'E'); - i = runEnd === -1 ? sql.length : runEnd; - out.push('?'); - clickhouseInsert.previous = ''; - continue; - } - - const number = isClickhouse ? matchClickhouseNumber(sql, i) : undefined; - if (number) { - out.push('?'); - i += number.length; - clickhouseInsert.previous = ''; - continue; - } - if (isClickhouse && isClickhouseInsertData(sql, i, char, clickhouseInsert)) { - break; - } - out.push(char); - i++; - } - - return out.join(''); -} - -/** - * Returns the character that closes an identifier opened by `char`, or undefined if `char` opens - * none. SQL Server brackets close on `]`, and `]]` escapes a `]` inside the name, which is the same - * doubling rule `findQuotedRunEnd` applies to quotes. - */ -function getIdentifierCloser(char: string, dialect: SqlDialect): string | undefined { - if (char === '`') { - return '`'; - } - if (char === '"' && dialect !== 'mysql') { - return '"'; - } - return char === '[' && dialect === 'mssql' ? ']' : undefined; -} - -/** Whether `char` can appear inside an identifier. `undefined` (start of query) counts as a break. */ -function isIdentifierChar(char: string | undefined): boolean { - return char !== undefined && /[\w$]/.test(char); -} - -/** Returns the opening dollar-quote tag at `start` (`$$` or `$tag$`), or undefined if there is none. */ -function matchDollarQuoteTag(sql: string, start: number, clickhouse: boolean): string | undefined { - const pattern = clickhouse ? CLICKHOUSE_DOLLAR_QUOTE_RE : DOLLAR_QUOTE_RE; - pattern.lastIndex = start; - return pattern.exec(sql)?.[0]; -} - -/** - * Returns the literal-prefix character immediately before a `'`, if there is one: `X`/`B` for - * hex/binary literals, `N` for a national-character literal (SQL Server, MySQL), or `E` for a - * PostgreSQL escape string (which honors backslash escapes). - */ -function getLiteralPrefix(out: string[], dialect: SqlDialect): 'X' | 'B' | 'N' | 'E' | undefined { - const last = out[out.length - 1]; - // A prefix only counts when it stands alone — the `X` in `MAX'...'` belongs to the identifier - if (last?.length !== 1 || isIdentifierChar(out[out.length - 2]?.slice(-1))) { - return undefined; - } - - const prefix = last.toUpperCase(); - if (prefix === 'X' || prefix === 'B' || prefix === 'N') { - return prefix; - } - return prefix === 'E' && dialect === 'standard' ? 'E' : undefined; -} - -/** - * Sanitize SQL query as per the OTEL semantic conventions - * https://opentelemetry.io/docs/specs/semconv/database/database-spans/#sanitization-of-dbquerytext - * - * Parameter placeholders survive: PostgreSQL `$n`, SQLite `?n`, and named forms like `:name` and - * `@name`. Per the OTEL spec they mark a parameterized query, so only values (strings, numbers, - * booleans) are sanitized. - * - * Pass `dialect` when the statement comes from a driver whose literals are not standard-quoted; - * see {@link SqlDialect}. - */ -export function sanitizeSqlQuery(sqlQuery: string | undefined, dialect: SqlDialect = 'standard'): string { - if (!sqlQuery) { - return 'Unknown SQL Query'; - } - - return ( - // Strip comments and string literals first: everything below is a regex that cannot tell - // whether it is looking at SQL syntax or at a user-supplied value. - stripLiteralsAndComments(sqlQuery, dialect) - .replace(/;\s*$/, '') // Remove trailing semicolons - .replace(/^[\s;]+/, '') // Remove leading semicolons (T-SQL's `;WITH`) - // Collapse whitespace to a single space (after removing comments) - .replace(/\s+/g, ' ') - .trim() // Remove extra spaces and trim - // Sanitize hex numbers - .replace(/\b0x[0-9A-Fa-f]+/gi, '?') - // Sanitize boolean literals - .replace(/\b(?:TRUE|FALSE)\b/gi, '?') - // Sanitize numeric literals (preserve $n placeholders via negative lookbehind) - .replace(/-?\b\d+\.?\d*[eE][+-]?\d+\b/g, '?') // Scientific notation - .replace(/-?\b\d+\.\d+\b/g, '?') // Decimals - .replace(/-?\.\d+\b/g, '?') // Decimals starting with dot - .replace(getIntegerLiteralRE(dialect), '?') // Integers (NOT parameter placeholders) - // Collapse IN clauses for cardinality (both ? and $n variants) - .replace(/\bIN\b\s*\(\s*\?(?:\s*,\s*\?)*\s*\)/gi, 'IN (?)') - .replace(/\bIN\b\s*\(\s*\$\d+(?:\s*,\s*\$\d+)*\s*\)/gi, 'IN ($?)') - ); -} diff --git a/packages/server-utils/src/utils/sql.ts b/packages/server-utils/src/utils/sql.ts index 4cee6a18dbe0..46d9221a4dad 100644 --- a/packages/server-utils/src/utils/sql.ts +++ b/packages/server-utils/src/utils/sql.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { DB_NAMESPACE, DB_OPERATION_NAME, @@ -7,11 +8,6 @@ import { } from '@sentry/conventions/attributes'; import type { SpanAttributes } from '@sentry/core'; import { getClient, hasSpanStreamingEnabled } from '@sentry/core'; -import type { SqlDialect } from './sql-sanitizer'; -import { sanitizeSqlQuery } from './sql-sanitizer'; - -export { sanitizeSqlQuery, toSqlDialect } from './sql-sanitizer'; -export type { SqlDialect } from './sql-sanitizer'; const MAX_SUMMARY_LENGTH = 255; @@ -34,7 +30,7 @@ const DDL_RE = new RegExp( // standalone shorthand. The clause is filler like INTO — stripping it keeps upserts on the same // low-cardinality summary as plain inserts. const INSERT_RE = new RegExp( - `^\\s*(?INSERT|REPLACE)(?:\\s+OR\\s+(?:ROLLBACK|ABORT|FAIL|IGNORE|REPLACE))?\\s+INTO\\s+(?
${TABLE_NAME})`, + `^\\s*(?INSERT|REPLACE)(?:\\s+OR\\s+(?:ROLLBACK|ABORT|FAIL|IGNORE|REPLACE))?\\s+INTO(?:\\s+TABLE)?\\s+(?
${TABLE_NAME})`, 'i', ); const UPDATE_RE = new RegExp( @@ -200,6 +196,374 @@ function truncate(summary: string): string { return lastSpace > 0 ? truncated.substring(0, lastSpace) : truncated; } +let integerLiteralRE: RegExp | undefined; +let mssqlIntegerLiteralRE: RegExp | undefined; + +/** + * The lookbehind keeps `$n` and `?n` parameter placeholders out of the match. T-SQL has neither, + * and `$1000` there is a money literal, so `$` only guards a placeholder outside `mssql`. + * + * Lazy init: constructing a lookbehind at module scope evaluates it on import and crashes + * Safari <16.4 browser bundles that reach this file via the core barrel. + */ +function getIntegerLiteralRE(dialect: SqlDialect): RegExp { + if (dialect === 'mssql') { + if (!mssqlIntegerLiteralRE) { + mssqlIntegerLiteralRE = new RegExp('(?', + '!', + '+', + '-', + '/', + '%', + '|', + '&', +]); + +interface ClickhouseInsertState { + /** Whether the statement is an INSERT, decided by its first word. */ + insert?: boolean; + /** The last token: an upper-cased word, a punctuation character, or `''` for a literal or quoted identifier. */ + previous?: string; + depth: number; +} + +/** + * Whether `char` starts the inline data of an INSERT. Everything after the top-level FORMAT clause is + * input data, which cannot be sanitized as SQL. That includes `INSERT ... SELECT`, where the + * `input()` table function reads that data. + */ +function isClickhouseInsertData(sql: string, start: number, char: string, state: ClickhouseInsertState): boolean { + if (char === '(') state.depth++; + if (char === ')') state.depth--; + if (/\s/.test(char) || (isIdentifierChar(char) && isIdentifierChar(sql[start - 1]))) { + return false; + } + CLICKHOUSE_WORD_RE.lastIndex = start; + const word = CLICKHOUSE_WORD_RE.exec(sql)?.[0].toUpperCase(); + if (word) { + state.insert ??= word === 'INSERT'; + } + if ( + state.insert && + state.depth === 0 && + word === 'FORMAT' && + !CLICKHOUSE_OPERAND_PREFIXES.has(state.previous ?? '') + ) { + return true; + } + state.previous = word ?? char; + return false; +} + +function matchClickhouseNumber(sql: string, start: number): string | undefined { + if (isIdentifierChar(sql[start - 1]) || sql[start - 1] === '?') { + return undefined; + } + CLICKHOUSE_NUMBER_RE.lastIndex = start; + return CLICKHOUSE_NUMBER_RE.exec(sql)?.[0]; +} + +/** + * Returns the index just past the run's closing `delimiter`, or -1 if the run never closes. The + * caller cannot derive that from the index alone: a query ending in `[a]]` consumed a doubled + * delimiter, not a closing one. + * + * A doubled delimiter (`''`) escapes itself in every dialect; backslash escapes are dialect- and + * context-dependent, so the caller decides. + */ +function findQuotedRunEnd(sql: string, start: number, delimiter: string, backslashEscapes: boolean): number { + for (let i = start + 1; i < sql.length; i++) { + const char = sql[i]; + if (backslashEscapes && char === '\\') { + i++; + } else if (char === delimiter) { + if (sql[i + 1] !== delimiter) { + return i + 1; + } + i++; + } + } + return -1; +} + +// An unterminated comment consumes the rest of the query. +function findBlockCommentEnd(sql: string, start: number, nested: boolean): number { + let depth = 1; + let i = start + 2; + while (i < sql.length && depth > 0) { + if (sql[i] === '*' && sql[i + 1] === '/') { + depth--; + i += 2; + } else if (nested && sql[i] === '/' && sql[i + 1] === '*') { + depth++; + i += 2; + } else { + i++; + } + } + return i; +} + +/** + * Replaces every string literal with `?` and drops every comment, in one pass. + * + * Doing this by scanning rather than by regex is what keeps quote state and comment state from + * being decided independently: a regex for `'...'` cannot see that the quote it stopped at was + * backslash-escaped, and a regex for `--...` cannot see that the `--` sits inside a literal. Both + * mistakes end with user data surviving into `db.query.text` and `db.query.summary`. + * + * Quoted identifiers are preserved — they are the table and column names the query summary is + * built from. + */ +function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { + const isMysql = dialect === 'mysql'; + const isClickhouse = dialect === 'clickhouse'; + const out: string[] = []; + let i = 0; + // A literal, number or quoted identifier completes an operand, so it resets `previous`. + const clickhouseInsert: ClickhouseInsertState = { depth: 0 }; + + while (i < sql.length) { + const char = sql[i]!; + const next = sql[i + 1]; + + if ((char === '-' && next === '-') || ((isMysql || isClickhouse) && char === '#')) { + const lineEnd = sql.indexOf('\n', i); + i = lineEnd === -1 ? sql.length : lineEnd; + continue; + } + + if (char === '/' && next === '*') { + i = findBlockCommentEnd(sql, i, isClickhouse); + continue; + } + + // In a dollar-quoted body (`$$body$$`, `$tag$body$tag$`) nothing has syntax meaning. Read the + // previous character from the query, not from `out`, where a dropped comment would leave `$$` + // looking like part of an identifier. Of the `standard` engines only PostgreSQL has dollar + // quoting. SQLite `$name` parameters do not match the tag pattern, apart from the rare `$a$b`. + // ClickHouse heredocs use the same syntax. + if ((dialect === 'standard' || isClickhouse) && char === '$' && !isIdentifierChar(sql[i - 1])) { + const tag = matchDollarQuoteTag(sql, i, isClickhouse); + if (tag) { + const bodyEnd = sql.indexOf(tag, i + tag.length); + i = bodyEnd === -1 ? sql.length : bodyEnd + tag.length; + out.push('?'); + clickhouseInsert.previous = ''; + continue; + } + } + + const identifierCloser = getIdentifierCloser(char, dialect); + if (identifierCloser) { + const runEnd = findQuotedRunEnd(sql, i, identifierCloser, isClickhouse); + // A run that never closes is not an identifier, so it collapses instead of being copied out. + // Copying would carry every literal in the rest of the statement through unlexed. + out.push(runEnd === -1 ? '?' : sql.slice(i, runEnd)); + i = runEnd === -1 ? sql.length : runEnd; + clickhouseInsert.previous = ''; + continue; + } + + if (char === "'" || (char === '"' && isMysql)) { + // A prefix like `X'1A'`, `B'01'`, `N'…'` or PostgreSQL's `E'a\nb'` is part of the literal, so it has + // to collapse into the same `?` instead of being left behind as a bare identifier. + const prefix = char === "'" ? getLiteralPrefix(out, dialect) : undefined; + if (prefix) { + out.pop(); + } + const runEnd = findQuotedRunEnd(sql, i, char, isMysql || isClickhouse || prefix === 'E'); + i = runEnd === -1 ? sql.length : runEnd; + out.push('?'); + clickhouseInsert.previous = ''; + continue; + } + + const number = isClickhouse ? matchClickhouseNumber(sql, i) : undefined; + if (number) { + out.push('?'); + i += number.length; + clickhouseInsert.previous = ''; + continue; + } + if (isClickhouse && isClickhouseInsertData(sql, i, char, clickhouseInsert)) { + break; + } + out.push(char); + i++; + } + + return out.join(''); +} + +/** + * Returns the character that closes an identifier opened by `char`, or undefined if `char` opens + * none. SQL Server brackets close on `]`, and `]]` escapes a `]` inside the name, which is the same + * doubling rule `findQuotedRunEnd` applies to quotes. + */ +function getIdentifierCloser(char: string, dialect: SqlDialect): string | undefined { + if (char === '`') { + return '`'; + } + if (char === '"' && dialect !== 'mysql') { + return '"'; + } + return char === '[' && dialect === 'mssql' ? ']' : undefined; +} + +/** Whether `char` can appear inside an identifier. `undefined` (start of query) counts as a break. */ +function isIdentifierChar(char: string | undefined): boolean { + return char !== undefined && /[\w$]/.test(char); +} + +/** Returns the opening dollar-quote tag at `start` (`$$` or `$tag$`), or undefined if there is none. */ +function matchDollarQuoteTag(sql: string, start: number, clickhouse: boolean): string | undefined { + const pattern = clickhouse ? CLICKHOUSE_DOLLAR_QUOTE_RE : DOLLAR_QUOTE_RE; + pattern.lastIndex = start; + return pattern.exec(sql)?.[0]; +} + +/** + * Returns the literal-prefix character immediately before a `'`, if there is one: `X`/`B` for + * hex/binary literals, `N` for a national-character literal (SQL Server, MySQL), or `E` for a + * PostgreSQL escape string (which honors backslash escapes). + */ +function getLiteralPrefix(out: string[], dialect: SqlDialect): 'X' | 'B' | 'N' | 'E' | undefined { + const last = out[out.length - 1]; + // A prefix only counts when it stands alone — the `X` in `MAX'...'` belongs to the identifier + if (last?.length !== 1 || isIdentifierChar(out[out.length - 2]?.slice(-1))) { + return undefined; + } + + const prefix = last.toUpperCase(); + if (prefix === 'X' || prefix === 'B' || prefix === 'N') { + return prefix; + } + return prefix === 'E' && dialect === 'standard' ? 'E' : undefined; +} + +/** + * Sanitize SQL query as per the OTEL semantic conventions + * https://opentelemetry.io/docs/specs/semconv/database/database-spans/#sanitization-of-dbquerytext + * + * Parameter placeholders survive: PostgreSQL `$n`, SQLite `?n`, and named forms like `:name` and + * `@name`. Per the OTEL spec they mark a parameterized query, so only values (strings, numbers, + * booleans) are sanitized. + * + * Pass `dialect` when the statement comes from a driver whose literals are not standard-quoted; + * see {@link SqlDialect}. + */ +export function sanitizeSqlQuery(sqlQuery: string | undefined, dialect: SqlDialect = 'standard'): string { + if (!sqlQuery) { + return 'Unknown SQL Query'; + } + + return ( + // Strip comments and string literals first: everything below is a regex that cannot tell + // whether it is looking at SQL syntax or at a user-supplied value. + stripLiteralsAndComments(sqlQuery, dialect) + .replace(/;\s*$/, '') // Remove trailing semicolons + .replace(/^[\s;]+/, '') // Remove leading semicolons (T-SQL's `;WITH`) + // Collapse whitespace to a single space (after removing comments) + .replace(/\s+/g, ' ') + .trim() // Remove extra spaces and trim + // Sanitize hex numbers + .replace(/\b0x[0-9A-Fa-f]+/gi, '?') + // Sanitize boolean literals + .replace(/\b(?:TRUE|FALSE)\b/gi, '?') + // Sanitize numeric literals (preserve $n placeholders via negative lookbehind) + .replace(/-?\b\d+\.?\d*[eE][+-]?\d+\b/g, '?') // Scientific notation + .replace(/-?\b\d+\.\d+\b/g, '?') // Decimals + .replace(/-?\.\d+\b/g, '?') // Decimals starting with dot + .replace(getIntegerLiteralRE(dialect), '?') // Integers (NOT parameter placeholders) + // Collapse IN clauses for cardinality (both ? and $n variants) + .replace(/\bIN\b\s*\(\s*\?(?:\s*,\s*\?)*\s*\)/gi, 'IN (?)') + .replace(/\bIN\b\s*\(\s*\$\d+(?:\s*,\s*\$\d+)*\s*\)/gi, 'IN ($?)') + ); +} + export interface SqlQueryAttributes { [DB_QUERY_TEXT]: string | undefined; [DB_QUERY_SUMMARY]: string | undefined; diff --git a/packages/server-utils/test/integrations/clickhouse.test.ts b/packages/server-utils/test/integrations/clickhouse.test.ts index a712b4cfc54c..67ed98372d98 100644 --- a/packages/server-utils/test/integrations/clickhouse.test.ts +++ b/packages/server-utils/test/integrations/clickhouse.test.ts @@ -113,7 +113,9 @@ describe('clickhouseIntegration', () => { beforeAll(() => { installTestAsyncContextStrategy(); - GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@clickhouse/client', '@clickhouse/client-common'] }; + GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { + runtime: ['@clickhouse/client', '@clickhouse/client-common', '@clickhouse/client-web'], + }; clickhouseIntegration().setup?.({ on: () => () => undefined } as unknown as Client); }); @@ -147,12 +149,12 @@ describe('clickhouseIntegration', () => { expect(startInactiveSpanSpy).toHaveBeenCalledWith({ name: 'SELECT events', onlyIfParent: true, - attributes: expect.objectContaining({ + attributes: { ...CONNECTION_ATTRIBUTES, [DB_QUERY_TEXT]: 'SELECT count() FROM events WHERE name = ? AND id > ?', [DB_QUERY_SUMMARY]: 'SELECT events', [DB_OPERATION_NAME]: 'SELECT', - }), + }, }); expect(span.end).toHaveBeenCalledTimes(1); }); @@ -168,12 +170,12 @@ describe('clickhouseIntegration', () => { expect(startInactiveSpanSpy).toHaveBeenCalledWith({ name: 'INSERT events', onlyIfParent: true, - attributes: expect.objectContaining({ + attributes: { ...CONNECTION_ATTRIBUTES, [DB_OPERATION_NAME]: 'INSERT', [DB_COLLECTION_NAME]: 'events', [DB_QUERY_SUMMARY]: 'INSERT events', - }), + }, }); // The inserted values never reach the span. expect(startInactiveSpanSpy.mock.calls[0]?.[0].attributes[DB_QUERY_TEXT]).toBeUndefined(); @@ -188,36 +190,50 @@ describe('clickhouseIntegration', () => { expect(startInactiveSpanSpy).not.toHaveBeenCalled(); }); - it('sets an error status on a failed call without capturing the error', async () => { - const ctx: ChannelContext = { arguments: [{ query: 'SELECT * FROM does_not_exist' }], self: CLIENT }; - - await expect( - tracingChannel(CHANNELS.CLICKHOUSE_QUERY).tracePromise(async () => { - throw new Error('Table analytics.does_not_exist does not exist'); - }, ctx), - ).rejects.toThrow('Table analytics.does_not_exist does not exist'); - - expect(span.setStatus).toHaveBeenCalledWith({ - code: SPAN_STATUS_ERROR, - message: 'Table analytics.does_not_exist does not exist', - }); - expect(span.end).toHaveBeenCalledTimes(1); - expect(captureExceptionSpy).not.toHaveBeenCalled(); - }); + it.each([CHANNELS.CLICKHOUSE_QUERY, CHANNELS.CLICKHOUSE_WEB_QUERY])( + 'sets an error status on a failed call without capturing the error (%s)', + async channel => { + const ctx: ChannelContext = { arguments: [{ query: 'SELECT * FROM does_not_exist' }], self: CLIENT }; + + await expect( + tracingChannel(channel).tracePromise(async () => { + throw new Error('Table analytics.does_not_exist does not exist'); + }, ctx), + ).rejects.toThrow('Table analytics.does_not_exist does not exist'); + + expect(span.setStatus).toHaveBeenCalledWith({ + code: SPAN_STATUS_ERROR, + message: 'Table analytics.does_not_exist does not exist', + }); + expect(span.end).toHaveBeenCalledTimes(1); + expect(captureExceptionSpy).not.toHaveBeenCalled(); + }, + ); it.each([ ['query', CHANNELS.CLICKHOUSE_COMMON_QUERY], ['command', CHANNELS.CLICKHOUSE_COMMON_COMMAND], ['exec', CHANNELS.CLICKHOUSE_COMMON_EXEC], ['insert', CHANNELS.CLICKHOUSE_COMMON_INSERT], - ])('builds a db span for `%s` from `@clickhouse/client-common`', async (_method, channel) => { + ['query', CHANNELS.CLICKHOUSE_WEB_QUERY], + ['command', CHANNELS.CLICKHOUSE_WEB_COMMAND], + ['exec', CHANNELS.CLICKHOUSE_WEB_EXEC], + ['insert', CHANNELS.CLICKHOUSE_WEB_INSERT], + ])('builds a db span for `%s` on %s', async (method, channel) => { const ctx: ChannelContext = { arguments: [{ query: 'SELECT 1', table: 'events', values: [1] }], self: CLIENT }; await tracingChannel(channel).tracePromise(async () => ({}), ctx); - expect(startInactiveSpanSpy).toHaveBeenCalledWith( - expect.objectContaining({ attributes: expect.objectContaining(CONNECTION_ATTRIBUTES) }), - ); + expect(startInactiveSpanSpy).toHaveBeenCalledWith({ + name: method === 'insert' ? 'INSERT events' : 'SELECT', + onlyIfParent: true, + attributes: { + ...CONNECTION_ATTRIBUTES, + ...(method === 'insert' + ? { [DB_OPERATION_NAME]: 'INSERT', [DB_COLLECTION_NAME]: 'events', [DB_QUERY_SUMMARY]: 'INSERT events' } + : { [DB_QUERY_TEXT]: 'SELECT ?', [DB_QUERY_SUMMARY]: 'SELECT', [DB_OPERATION_NAME]: 'SELECT' }), + }, + }); expect(span.end).toHaveBeenCalledTimes(1); }); @@ -230,15 +246,20 @@ describe('clickhouseIntegration', () => { await tracingChannel(CHANNELS.CLICKHOUSE_COMMON_QUERY).tracePromise(async () => ({}), ctx); - expect(startInactiveSpanSpy).toHaveBeenCalledWith( - expect.objectContaining({ - attributes: expect.objectContaining({ - [DB_USER]: 'default', - [SERVER_ADDRESS]: 'ch.example.com', - [SERVER_PORT]: 443, - }), - }), - ); + expect(startInactiveSpanSpy).toHaveBeenCalledWith({ + name: 'SELECT', + onlyIfParent: true, + attributes: { + ...CONNECTION_ATTRIBUTES, + [DB_NAMESPACE]: 'default', + [DB_USER]: 'default', + [SERVER_ADDRESS]: 'ch.example.com', + [SERVER_PORT]: 443, + [DB_QUERY_TEXT]: 'SELECT ?', + [DB_QUERY_SUMMARY]: 'SELECT', + [DB_OPERATION_NAME]: 'SELECT', + }, + }); }); it('prefers a per-call `auth` override for the username', async () => { @@ -249,8 +270,16 @@ describe('clickhouseIntegration', () => { await tracingChannel(CHANNELS.CLICKHOUSE_QUERY).tracePromise(async () => ({}), ctx); - expect(startInactiveSpanSpy).toHaveBeenCalledWith( - expect.objectContaining({ attributes: expect.objectContaining({ [DB_USER]: 'writer' }) }), - ); + expect(startInactiveSpanSpy).toHaveBeenCalledWith({ + name: 'SELECT', + onlyIfParent: true, + attributes: { + ...CONNECTION_ATTRIBUTES, + [DB_USER]: 'writer', + [DB_QUERY_TEXT]: 'SELECT ?', + [DB_QUERY_SUMMARY]: 'SELECT', + [DB_OPERATION_NAME]: 'SELECT', + }, + }); }); }); diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index 1af8376c0fdd..75c9f5b23878 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -85,6 +85,10 @@ describe('getSqlQuerySummary', () => { it.each([ ['INSERT INTO users (name, email) VALUES (?, ?)', 'INSERT users'], ['insert into orders (product_id) values (?)', 'insert orders'], + ['INSERT INTO TABLE events VALUES (?)', 'INSERT events'], + ['insert into table analytics.events values (?)', 'insert analytics.events'], + ['INSERT INTO TABLE "events" VALUES (?)', 'INSERT "events"'], + ['INSERT INTO TABLE events SELECT * FROM source', 'INSERT events SELECT source'], ])('strips INTO: %j => %j', (input, expected) => { expect(getSqlQuerySummary(input)).toBe(expected); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 59df360d50ca..797ec72721dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -481,6 +481,9 @@ importers: '@anthropic-ai/sdk': specifier: 0.133.0 version: 0.133.0(zod@4.6.7) + '@clickhouse/client-web': + specifier: ^1.25.0 + version: 1.25.0 '@cloudflare/vite-plugin': specifier: 1.66.0 version: 1.66.0(@types/node@20.19.43)(vite@8.3.6(@types/node@20.19.43)(esbuild@0.28.1)(jiti@2.7.0)(less@4.2.0(supports-color@10.0.0))(sass@1.72.0)(stylus@0.59.0(supports-color@10.0.0))(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.0))(workerd@1.20261023.1)(wrangler@4.156.0(@cloudflare/workers-types@4.20260519.1)(@types/node@20.19.43)) @@ -619,7 +622,7 @@ importers: specifier: ^3.1161.0 version: 3.1161.0 '@clickhouse/client': - specifier: 1.25.0 + specifier: ^1.25.0 version: 1.25.0 '@earendil-works/chord': specifier: ^1.4.0 @@ -4156,6 +4159,9 @@ packages: resolution: {integrity: sha512-dlT1m5e/0yUL0kRNcQn7yGLVThkgbB0Ga/1AmfDDC/8ik6AIiSf2QLQO2zPYvefsHP0aFgxO93cVLCCfDp7kzQ==} engines: {node: '>= 20.12.0'} + '@clickhouse/client-web@1.25.0': + resolution: {integrity: sha512-9XgfE82bp8yZBMHql2JrcQz3qnNIuyscChkaSeilXcPl14BCJPdBmuV7oY8AOev9K0LNbMGRD+5GNJAZVSlveg==} + '@clickhouse/client@1.25.0': resolution: {integrity: sha512-sxoCDlA8iOEGpGrJzl9SeNvG7II6e5kSeo0JI4Mru2QhwhjydWuDEE55ryZg3vIJ0CSJ1bhn9+Xtnuvf1BZT/w==} engines: {node: '>=20'} @@ -22016,6 +22022,8 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 + '@clickhouse/client-web@1.25.0': {} + '@clickhouse/client@1.25.0': {} '@cloudflare/kv-asset-handler@0.4.2': {} From 03055eb104778ee8a3f0420816c0f7ba8c073a72 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Fri, 20 Nov 2026 12:38:36 +0100 Subject: [PATCH 6/8] fixup! feat(node): add ClickHouse instrumentation --- .../suites/tracing/clickhouse/test.ts | 9 +- .../src/integrations/clickhouse-sql.ts | 39 ++++ .../src/integrations/clickhouse.ts | 3 +- packages/server-utils/src/utils/sql.ts | 182 ++++++------------ packages/server-utils/test/utils/sql.test.ts | 8 +- 5 files changed, 106 insertions(+), 135 deletions(-) create mode 100644 packages/server-utils/src/integrations/clickhouse-sql.ts diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts index 1faf6cb17959..3942eee05c82 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/clickhouse/test.ts @@ -12,6 +12,7 @@ import { SERVER_ADDRESS, SERVER_PORT, } from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { expect, it } from 'vitest'; import { createRunner } from '../../../runner'; import { getSpansFromEnvelope } from '../../../spanUtils'; @@ -32,7 +33,7 @@ it('instruments client-web calls in a bundled Worker and leaves handled errors a for (const span of dbSpans) { expect(span.parent_span_id).toBe(segment?.span_id); expect(span.trace_id).toBe(segment?.trace_id); - expect(span.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'db' }); + expect(span.attributes[SENTRY_OP]).toEqual({ type: 'string', value: DB }); expect(span.attributes[SENTRY_KIND]).toEqual({ type: 'string', value: 'client' }); expect(span.attributes[DB_SYSTEM_NAME]).toEqual({ type: 'string', value: 'clickhouse' }); expect(span.attributes[DB_NAMESPACE]).toEqual({ type: 'string', value: 'tests' }); @@ -105,7 +106,7 @@ it('instruments client-web calls in a bundled Worker and leaves handled errors a const envelopes = runner.getReceivedEnvelopes(); expect(envelopes.some(envelope => envelope[1].some(item => item[0].type === 'event'))).toBe(false); - expect( - envelopes.flatMap(getSpansFromEnvelope).filter(span => span.attributes[SENTRY_OP]?.value === 'db'), - ).toHaveLength(8); + expect(envelopes.flatMap(getSpansFromEnvelope).filter(span => span.attributes[SENTRY_OP]?.value === DB)).toHaveLength( + 8, + ); }); diff --git a/packages/server-utils/src/integrations/clickhouse-sql.ts b/packages/server-utils/src/integrations/clickhouse-sql.ts new file mode 100644 index 000000000000..44fa0ea3971e --- /dev/null +++ b/packages/server-utils/src/integrations/clickhouse-sql.ts @@ -0,0 +1,39 @@ +import type { SqlLexerRules } from '../utils/sql'; +import { isIdentifierChar } from '../utils/sql'; + +const NUMBER_RE = /-?(?:0x[\dA-F_]+|0b[01_]+|(?:\d[\d_]*(?:\.[\d_]*)?|\.\d[\d_]*)(?:e[+-]?[\d_]+)?)(?![\w$])/iy; +const FORMAT_RE = /FORMAT(?:\s|\/\*|--|#)/iy; +// After one of these tokens, `format` is a table, column or alias name. +const FORMAT_IDENTIFIER_RE = /(?:\b(?:INTO|TABLE|FROM|JOIN|SELECT|WHERE|AND|OR|BY|AS)|[,.=<>(])\s*$/i; + +/** + * ClickHouse SQL: `"` quotes an identifier as in `standard`, but `\` escapes the next character, `#` + * starts a line comment, block comments nest, and `$$…$$` heredocs open a literal. Numbers can have + * `_` separators. + */ +export const CLICKHOUSE_SQL_RULES: SqlLexerRules = { + hashComments: true, + nestedComments: true, + backslashEscapes: true, + dollarQuote: /\$\w*\$/y, + matchLiteral(sql, start) { + if (isIdentifierChar(sql[start - 1]) || sql[start - 1] === '?') { + return 0; + } + NUMBER_RE.lastIndex = start; + return NUMBER_RE.exec(sql)?.[0].length ?? 0; + }, + // Everything after the FORMAT clause of an INSERT is inline data. That includes `INSERT ... SELECT`, + // where the `input()` table function reads that data. + isInlineData(sql, start, out) { + if (isIdentifierChar(sql[start - 1])) { + return false; + } + FORMAT_RE.lastIndex = start; + if (!FORMAT_RE.test(sql)) { + return false; + } + const before = out.join(''); + return /^[\s;]*INSERT\b/i.test(before) && !FORMAT_IDENTIFIER_RE.test(before); + }, +}; diff --git a/packages/server-utils/src/integrations/clickhouse.ts b/packages/server-utils/src/integrations/clickhouse.ts index 78e50c04da5e..a19b591650bd 100644 --- a/packages/server-utils/src/integrations/clickhouse.ts +++ b/packages/server-utils/src/integrations/clickhouse.ts @@ -21,6 +21,7 @@ import { clickhouseModuleNames } from '../orchestrion/config/clickhouse'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindTracingChannelToSpan } from '../tracing-channel'; import { getLowCardinalitySqlSpanName, getSqlQueryAttributes } from '../utils/sql'; +import { CLICKHOUSE_SQL_RULES } from './clickhouse-sql'; const INTEGRATION_NAME = 'ClickHouse' as const; const ORIGIN = 'auto.db.clickhouse'; @@ -100,7 +101,7 @@ function instrumentClickhouse(): void { ...getConnectionAttributes(data.self?.connectionParams, params), ...(method === 'insert' ? getInsertAttributes(params) - : getSqlQueryAttributes(typeof params?.query === 'string' ? params.query : undefined, 'clickhouse')), + : getSqlQueryAttributes(typeof params?.query === 'string' ? params.query : undefined, CLICKHOUSE_SQL_RULES)), }; return startInactiveSpan({ diff --git a/packages/server-utils/src/utils/sql.ts b/packages/server-utils/src/utils/sql.ts index 46d9221a4dad..77f86d79ae3b 100644 --- a/packages/server-utils/src/utils/sql.ts +++ b/packages/server-utils/src/utils/sql.ts @@ -206,7 +206,7 @@ let mssqlIntegerLiteralRE: RegExp | undefined; * Lazy init: constructing a lookbehind at module scope evaluates it on import and crashes * Safari <16.4 browser bundles that reach this file via the core barrel. */ -function getIntegerLiteralRE(dialect: SqlDialect): RegExp { +function getIntegerLiteralRE(dialect: SqlDialect | SqlLexerRules): RegExp { if (dialect === 'mssql') { if (!mssqlIntegerLiteralRE) { mssqlIntegerLiteralRE = new RegExp('(? number; + /** + * Returns true when inline data starts at `start`. The lexer drops the query text from `start` to the + * end, because data cannot be sanitized as SQL. `out` holds the sanitized text before `start`. + */ + isInlineData?: (sql: string, start: number, out: string[]) => boolean; +} /** * Maps a driver or `db.system.name` value to the dialect its statements are written in. Callers @@ -245,105 +267,13 @@ export function toSqlDialect(system: unknown): SqlDialect { if (system === 'mssql' || system === 'sqlserver' || system === 'microsoft.sql_server') { return 'mssql'; } - if (system === 'clickhouse') { - return 'clickhouse'; - } return 'standard'; } +const NO_RULES: SqlLexerRules = {}; + // Sticky, so the scanner can test one position without slicing the query on every `$`. const DOLLAR_QUOTE_RE = /\$(?:[A-Za-z_]\w*)?\$/y; -const CLICKHOUSE_DOLLAR_QUOTE_RE = /\$\w*\$/y; -const CLICKHOUSE_NUMBER_RE = - /-?(?:0x[\dA-F_]+|0b[01_]+|(?:\d[\d_]*(?:\.[\d_]*)?|\.\d[\d_]*)(?:e[+-]?[\d_]+)?)(?![\w$])/iy; - -const CLICKHOUSE_WORD_RE = /[A-Za-z_][\w$]*/y; - -// A FORMAT clause follows a complete table name or expression. After one of these tokens `format` -// is a table, column or alias name instead. -const CLICKHOUSE_OPERAND_PREFIXES = new Set([ - 'INTO', - 'TABLE', - 'SELECT', - 'DISTINCT', - 'FROM', - 'JOIN', - 'USING', - 'ON', - 'WHERE', - 'PREWHERE', - 'HAVING', - 'BY', - 'AND', - 'OR', - 'NOT', - 'AS', - 'CASE', - 'WHEN', - 'THEN', - 'ELSE', - 'IN', - 'IS', - 'LIKE', - 'ILIKE', - 'BETWEEN', - ',', - '.', - '=', - '<', - '>', - '!', - '+', - '-', - '/', - '%', - '|', - '&', -]); - -interface ClickhouseInsertState { - /** Whether the statement is an INSERT, decided by its first word. */ - insert?: boolean; - /** The last token: an upper-cased word, a punctuation character, or `''` for a literal or quoted identifier. */ - previous?: string; - depth: number; -} - -/** - * Whether `char` starts the inline data of an INSERT. Everything after the top-level FORMAT clause is - * input data, which cannot be sanitized as SQL. That includes `INSERT ... SELECT`, where the - * `input()` table function reads that data. - */ -function isClickhouseInsertData(sql: string, start: number, char: string, state: ClickhouseInsertState): boolean { - if (char === '(') state.depth++; - if (char === ')') state.depth--; - if (/\s/.test(char) || (isIdentifierChar(char) && isIdentifierChar(sql[start - 1]))) { - return false; - } - CLICKHOUSE_WORD_RE.lastIndex = start; - const word = CLICKHOUSE_WORD_RE.exec(sql)?.[0].toUpperCase(); - if (word) { - state.insert ??= word === 'INSERT'; - } - if ( - state.insert && - state.depth === 0 && - word === 'FORMAT' && - !CLICKHOUSE_OPERAND_PREFIXES.has(state.previous ?? '') - ) { - return true; - } - state.previous = word ?? char; - return false; -} - -function matchClickhouseNumber(sql: string, start: number): string | undefined { - if (isIdentifierChar(sql[start - 1]) || sql[start - 1] === '?') { - return undefined; - } - CLICKHOUSE_NUMBER_RE.lastIndex = start; - return CLICKHOUSE_NUMBER_RE.exec(sql)?.[0]; -} /** * Returns the index just past the run's closing `delimiter`, or -1 if the run never closes. The @@ -397,26 +327,26 @@ function findBlockCommentEnd(sql: string, start: number, nested: boolean): numbe * Quoted identifiers are preserved — they are the table and column names the query summary is * built from. */ -function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { - const isMysql = dialect === 'mysql'; - const isClickhouse = dialect === 'clickhouse'; +function stripLiteralsAndComments(sql: string, dialect: SqlDialect | SqlLexerRules): string { + const [base, rules]: [SqlDialect, SqlLexerRules] = + typeof dialect === 'string' ? [dialect, NO_RULES] : ['standard', dialect]; + const isMysql = base === 'mysql'; + const backslashEscapes = !!rules.backslashEscapes; const out: string[] = []; let i = 0; - // A literal, number or quoted identifier completes an operand, so it resets `previous`. - const clickhouseInsert: ClickhouseInsertState = { depth: 0 }; while (i < sql.length) { const char = sql[i]!; const next = sql[i + 1]; - if ((char === '-' && next === '-') || ((isMysql || isClickhouse) && char === '#')) { + if ((char === '-' && next === '-') || ((isMysql || rules.hashComments) && char === '#')) { const lineEnd = sql.indexOf('\n', i); i = lineEnd === -1 ? sql.length : lineEnd; continue; } if (char === '/' && next === '*') { - i = findBlockCommentEnd(sql, i, isClickhouse); + i = findBlockCommentEnd(sql, i, !!rules.nestedComments); continue; } @@ -424,51 +354,46 @@ function stripLiteralsAndComments(sql: string, dialect: SqlDialect): string { // previous character from the query, not from `out`, where a dropped comment would leave `$$` // looking like part of an identifier. Of the `standard` engines only PostgreSQL has dollar // quoting. SQLite `$name` parameters do not match the tag pattern, apart from the rare `$a$b`. - // ClickHouse heredocs use the same syntax. - if ((dialect === 'standard' || isClickhouse) && char === '$' && !isIdentifierChar(sql[i - 1])) { - const tag = matchDollarQuoteTag(sql, i, isClickhouse); + if (base === 'standard' && char === '$' && !isIdentifierChar(sql[i - 1])) { + const tag = matchDollarQuoteTag(sql, i, rules.dollarQuote); if (tag) { const bodyEnd = sql.indexOf(tag, i + tag.length); i = bodyEnd === -1 ? sql.length : bodyEnd + tag.length; out.push('?'); - clickhouseInsert.previous = ''; continue; } } - const identifierCloser = getIdentifierCloser(char, dialect); + const identifierCloser = getIdentifierCloser(char, base); if (identifierCloser) { - const runEnd = findQuotedRunEnd(sql, i, identifierCloser, isClickhouse); + const runEnd = findQuotedRunEnd(sql, i, identifierCloser, backslashEscapes); // A run that never closes is not an identifier, so it collapses instead of being copied out. // Copying would carry every literal in the rest of the statement through unlexed. out.push(runEnd === -1 ? '?' : sql.slice(i, runEnd)); i = runEnd === -1 ? sql.length : runEnd; - clickhouseInsert.previous = ''; continue; } if (char === "'" || (char === '"' && isMysql)) { // A prefix like `X'1A'`, `B'01'`, `N'…'` or PostgreSQL's `E'a\nb'` is part of the literal, so it has // to collapse into the same `?` instead of being left behind as a bare identifier. - const prefix = char === "'" ? getLiteralPrefix(out, dialect) : undefined; + const prefix = char === "'" ? getLiteralPrefix(out, base) : undefined; if (prefix) { out.pop(); } - const runEnd = findQuotedRunEnd(sql, i, char, isMysql || isClickhouse || prefix === 'E'); + const runEnd = findQuotedRunEnd(sql, i, char, isMysql || backslashEscapes || prefix === 'E'); i = runEnd === -1 ? sql.length : runEnd; out.push('?'); - clickhouseInsert.previous = ''; continue; } - const number = isClickhouse ? matchClickhouseNumber(sql, i) : undefined; - if (number) { + const literalLength = rules.matchLiteral?.(sql, i); + if (literalLength) { out.push('?'); - i += number.length; - clickhouseInsert.previous = ''; + i += literalLength; continue; } - if (isClickhouse && isClickhouseInsertData(sql, i, char, clickhouseInsert)) { + if (rules.isInlineData?.(sql, i, out)) { break; } out.push(char); @@ -494,13 +419,12 @@ function getIdentifierCloser(char: string, dialect: SqlDialect): string | undefi } /** Whether `char` can appear inside an identifier. `undefined` (start of query) counts as a break. */ -function isIdentifierChar(char: string | undefined): boolean { +export function isIdentifierChar(char: string | undefined): boolean { return char !== undefined && /[\w$]/.test(char); } /** Returns the opening dollar-quote tag at `start` (`$$` or `$tag$`), or undefined if there is none. */ -function matchDollarQuoteTag(sql: string, start: number, clickhouse: boolean): string | undefined { - const pattern = clickhouse ? CLICKHOUSE_DOLLAR_QUOTE_RE : DOLLAR_QUOTE_RE; +function matchDollarQuoteTag(sql: string, start: number, pattern = DOLLAR_QUOTE_RE): string | undefined { pattern.lastIndex = start; return pattern.exec(sql)?.[0]; } @@ -533,9 +457,12 @@ function getLiteralPrefix(out: string[], dialect: SqlDialect): 'X' | 'B' | 'N' | * booleans) are sanitized. * * Pass `dialect` when the statement comes from a driver whose literals are not standard-quoted; - * see {@link SqlDialect}. + * see {@link SqlDialect} and {@link SqlLexerRules}. */ -export function sanitizeSqlQuery(sqlQuery: string | undefined, dialect: SqlDialect = 'standard'): string { +export function sanitizeSqlQuery( + sqlQuery: string | undefined, + dialect: SqlDialect | SqlLexerRules = 'standard', +): string { if (!sqlQuery) { return 'Unknown SQL Query'; } @@ -576,7 +503,10 @@ export interface SqlQueryAttributes { * when there is no statement, so an empty query omits the attributes instead of reporting the * sanitizer's fallback text. */ -export function getSqlQueryAttributes(sqlQuery: string | undefined, dialect?: SqlDialect): SqlQueryAttributes { +export function getSqlQueryAttributes( + sqlQuery: string | undefined, + dialect?: SqlDialect | SqlLexerRules, +): SqlQueryAttributes { const queryText = (sqlQuery && sanitizeSqlQuery(sqlQuery, dialect)) || undefined; return { [DB_QUERY_TEXT]: queryText, diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index 75c9f5b23878..faf217416998 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -7,6 +7,7 @@ import { } from '@sentry/conventions/attributes'; import { setCurrentClient } from '@sentry/core'; import { afterEach, describe, expect, it } from 'vitest'; +import { CLICKHOUSE_SQL_RULES } from '../../src/integrations/clickhouse-sql'; import { getSqlOperationName, getSqlQuerySummary, @@ -773,7 +774,7 @@ describe('sanitizeSqlQuery', () => { }); }); - describe("dialect: 'clickhouse'", () => { + describe('CLICKHOUSE_SQL_RULES', () => { it.each([ ['SELECT 1 /* outer /* inner */ private@example.com */', 'SELECT ?'], ['SELECT /* outer /* inner */ private@example.com */ 1', 'SELECT ?'], @@ -787,7 +788,7 @@ describe('sanitizeSqlQuery', () => { 'INSERT INTO events', ], ])('removes the entire nested comment in %p', (input, expected) => { - expect(sanitizeSqlQuery(input, 'clickhouse')).toBe(expected); + expect(sanitizeSqlQuery(input, CLICKHOUSE_SQL_RULES)).toBe(expected); }); it.each([ @@ -840,7 +841,7 @@ describe('sanitizeSqlQuery', () => { 'SELECT * FROM t WHERE id = {id:UInt32} AND name = {name:String}', ], ])('sanitizes %p', (input, expected) => { - expect(sanitizeSqlQuery(input, 'clickhouse')).toBe(expected); + expect(sanitizeSqlQuery(input, CLICKHOUSE_SQL_RULES)).toBe(expected); }); }); @@ -988,7 +989,6 @@ describe('toSqlDialect', () => { ['mssql', 'mssql'], ['sqlserver', 'mssql'], ['microsoft.sql_server', 'mssql'], - ['clickhouse', 'clickhouse'], ])('maps %j to %j', (system, expected) => { expect(toSqlDialect(system)).toBe(expected); }); From c39ceac1af5c0d0f86dc8d36357717b3a43b0284 Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:26:01 -0600 Subject: [PATCH 7/8] fix(node): preserve ClickHouse FORMAT comparisons --- .../src/integrations/clickhouse-sql.ts | 3 ++- packages/server-utils/test/utils/sql.test.ts | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/server-utils/src/integrations/clickhouse-sql.ts b/packages/server-utils/src/integrations/clickhouse-sql.ts index 44fa0ea3971e..ddd9ab283af2 100644 --- a/packages/server-utils/src/integrations/clickhouse-sql.ts +++ b/packages/server-utils/src/integrations/clickhouse-sql.ts @@ -2,7 +2,8 @@ import type { SqlLexerRules } from '../utils/sql'; import { isIdentifierChar } from '../utils/sql'; const NUMBER_RE = /-?(?:0x[\dA-F_]+|0b[01_]+|(?:\d[\d_]*(?:\.[\d_]*)?|\.\d[\d_]*)(?:e[+-]?[\d_]+)?)(?![\w$])/iy; -const FORMAT_RE = /FORMAT(?:\s|\/\*|--|#)/iy; +// A comparison operator can follow a column named `format`, but cannot start a format name. +const FORMAT_RE = /FORMAT(?!\s*[=<>!])(?:\s|\/\*|--|#)/iy; // After one of these tokens, `format` is a table, column or alias name. const FORMAT_IDENTIFIER_RE = /(?:\b(?:INTO|TABLE|FROM|JOIN|SELECT|WHERE|AND|OR|BY|AS)|[,.=<>(])\s*$/i; diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index faf217416998..67e1e7a16d0f 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -821,6 +821,22 @@ describe('sanitizeSqlQuery', () => { 'INSERT INTO events SELECT * FROM source WHERE format = 1 ORDER BY format', 'INSERT INTO events SELECT * FROM source WHERE format = ? ORDER BY format', ], + [ + 'INSERT INTO events SELECT a.id FROM source a JOIN lookup b ON format = b.kind', + 'INSERT INTO events SELECT a.id FROM source a JOIN lookup b ON format = b.kind', + ], + [ + 'INSERT INTO events SELECT id FROM source PREWHERE format >= 2', + 'INSERT INTO events SELECT id FROM source PREWHERE format >= ?', + ], + [ + 'INSERT INTO events SELECT format FROM source GROUP BY format HAVING format > 2', + 'INSERT INTO events SELECT format FROM source GROUP BY format HAVING format > ?', + ], + [ + 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format <= 2', + 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format <= ?', + ], ['INSERT INTO TABLE format FORMAT CSV\nprivate@example.com', 'INSERT INTO TABLE format'], ["INSERT INTO events (`format`) VALUES ('private@example.com')", 'INSERT INTO events (`format`) VALUES (?)'], // `"..."` and backticks quote identifiers, as in `standard` From 8f80dc9b93b20c25b8a43977c6edb3a0d153adec Mon Sep 17 00:00:00 2001 From: Aditya Reddy <82123567+adityareddy-dev@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:43:40 -0600 Subject: [PATCH 8/8] fix(node): Keep ClickHouse format columns after ON, PREWHERE, HAVING, QUALIFY and NOT A column named `format` after these keywords was read as the FORMAT clause of an INSERT, so the query text was cut off there. They now count as tokens after which `format` is a name, like WHERE and AND. --- .../src/integrations/clickhouse-sql.ts | 3 ++- packages/server-utils/test/utils/sql.test.ts | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/server-utils/src/integrations/clickhouse-sql.ts b/packages/server-utils/src/integrations/clickhouse-sql.ts index ddd9ab283af2..5fd54df3de9d 100644 --- a/packages/server-utils/src/integrations/clickhouse-sql.ts +++ b/packages/server-utils/src/integrations/clickhouse-sql.ts @@ -5,7 +5,8 @@ const NUMBER_RE = /-?(?:0x[\dA-F_]+|0b[01_]+|(?:\d[\d_]*(?:\.[\d_]*)?|\.\d[\d_]* // A comparison operator can follow a column named `format`, but cannot start a format name. const FORMAT_RE = /FORMAT(?!\s*[=<>!])(?:\s|\/\*|--|#)/iy; // After one of these tokens, `format` is a table, column or alias name. -const FORMAT_IDENTIFIER_RE = /(?:\b(?:INTO|TABLE|FROM|JOIN|SELECT|WHERE|AND|OR|BY|AS)|[,.=<>(])\s*$/i; +const FORMAT_IDENTIFIER_RE = + /(?:\b(?:INTO|TABLE|FROM|JOIN|ON|SELECT|PREWHERE|WHERE|HAVING|QUALIFY|AND|OR|NOT|BY|AS)|[,.=<>(])\s*$/i; /** * ClickHouse SQL: `"` quotes an identifier as in `standard`, but `\` escapes the next character, `#` diff --git a/packages/server-utils/test/utils/sql.test.ts b/packages/server-utils/test/utils/sql.test.ts index 67e1e7a16d0f..8f4ef94cfd63 100644 --- a/packages/server-utils/test/utils/sql.test.ts +++ b/packages/server-utils/test/utils/sql.test.ts @@ -837,6 +837,26 @@ describe('sanitizeSqlQuery', () => { 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format <= 2', 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format <= ?', ], + [ + 'INSERT INTO events SELECT a.id FROM source a JOIN lookup b ON format IS NOT NULL', + 'INSERT INTO events SELECT a.id FROM source a JOIN lookup b ON format IS NOT NULL', + ], + [ + "INSERT INTO events SELECT id FROM source PREWHERE format LIKE 'json%'", + 'INSERT INTO events SELECT id FROM source PREWHERE format LIKE ?', + ], + [ + 'INSERT INTO events SELECT format FROM source GROUP BY format HAVING format IN (1, 2)', + 'INSERT INTO events SELECT format FROM source GROUP BY format HAVING format IN (?)', + ], + [ + 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format BETWEEN 1 AND 2', + 'INSERT INTO events SELECT row_number() OVER () AS format FROM source QUALIFY format BETWEEN ? AND ?', + ], + [ + 'INSERT INTO events SELECT id FROM source WHERE NOT format IN (1, 2)', + 'INSERT INTO events SELECT id FROM source WHERE NOT format IN (?)', + ], ['INSERT INTO TABLE format FORMAT CSV\nprivate@example.com', 'INSERT INTO TABLE format'], ["INSERT INTO events (`format`) VALUES ('private@example.com')", 'INSERT INTO events (`format`) VALUES (?)'], // `"..."` and backticks quote identifiers, as in `standard`