From 749b103cb1c4aeb26847be9363725f513dcc1dbb Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 2 Jun 2026 18:52:16 +0800 Subject: [PATCH 01/24] [improvement](fe) Add external table metadata profile details (#63648) Problem Summary: External table queries previously showed only coarse FE metadata time in profile, making it hard to locate slow metadata access steps for Hive, Iceberg, Hudi, and Paimon scans. This change records dedicated profile timings for external partition value loading, partition metadata loading, partition file listing, and file scan task planning. The scan nodes keep a SummaryProfile reference so asynchronous split planning can also report its metadata time. --- .../doris/common/profile/SummaryProfile.java | 92 +++++++++- .../doris/datasource/FileQueryScanNode.java | 14 +- .../datasource/hive/HMSExternalTable.java | 28 ++- .../datasource/hive/source/HiveScanNode.java | 83 +++++---- .../doris/datasource/hudi/HudiUtils.java | 77 ++++---- .../datasource/hudi/source/HudiScanNode.java | 167 ++++++++++-------- .../iceberg/source/IcebergApiSource.java | 25 +-- .../iceberg/source/IcebergHMSSource.java | 10 +- .../iceberg/source/IcebergScanNode.java | 82 ++++++--- .../paimon/source/PaimonScanNode.java | 49 +++-- .../common/profile/SummaryProfileTest.java | 24 +++ 11 files changed, 451 insertions(+), 200 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index bbc7e4994094ce..f59d9a03da9f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -82,8 +82,12 @@ public class SummaryProfile { public static final String INIT_SCAN_NODE_TIME = "Init Scan Node Time"; public static final String FINALIZE_SCAN_NODE_TIME = "Finalize Scan Node Time"; public static final String GET_SPLITS_TIME = "Get Splits Time"; + public static final String EXTERNAL_TABLE_META_TIME = "External Table Meta Time"; + public static final String EXTERNAL_TABLE_GET_TABLE_META_TIME = "External Table Get Table Meta Time"; + public static final String EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME = "External Table Get Partition Values Time"; public static final String GET_PARTITIONS_TIME = "Get Partitions Time"; public static final String GET_PARTITION_FILES_TIME = "Get Partition Files Time"; + public static final String EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME = "External Table Get File Scan Tasks Time"; public static final String CREATE_SCAN_RANGE_TIME = "Create Scan Range Time"; public static final String SINK_SET_PARTITION_VALUES_TIME = "Sink Set Partition Values Time"; public static final String PLAN_TIME = "Plan Time"; @@ -173,8 +177,12 @@ public boolean isWarmup() { INIT_SCAN_NODE_TIME, FINALIZE_SCAN_NODE_TIME, GET_SPLITS_TIME, + EXTERNAL_TABLE_META_TIME, + EXTERNAL_TABLE_GET_TABLE_META_TIME, + EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, GET_PARTITIONS_TIME, GET_PARTITION_FILES_TIME, + EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, SINK_SET_PARTITION_VALUES_TIME, CREATE_SCAN_RANGE_TIME, ICEBERG_SCAN_METRICS, @@ -225,10 +233,14 @@ public boolean isWarmup() { .put(INIT_SCAN_NODE_TIME, 2) .put(FINALIZE_SCAN_NODE_TIME, 2) .put(GET_SPLITS_TIME, 3) + .put(EXTERNAL_TABLE_META_TIME, 4) + .put(EXTERNAL_TABLE_GET_TABLE_META_TIME, 5) + .put(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, 5) .put(NEREIDS_DISTRIBUTE_TIME, 1) .put(NEREIDS_BE_FOLD_CONST_TIME, 2) .put(GET_PARTITIONS_TIME, 3) .put(GET_PARTITION_FILES_TIME, 3) + .put(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, 5) .put(SINK_SET_PARTITION_VALUES_TIME, 3) .put(CREATE_SCAN_RANGE_TIME, 2) .put(ICEBERG_SCAN_METRICS, 3) @@ -379,6 +391,26 @@ public boolean isWarmup() { private long nereidsMvRewriteTime = 0; @SerializedName(value = "externalCatalogMetaTime") private long externalCatalogMetaTime = 0; + // Total time to get table meta, including time to get table meta from external catalog and time to do some + // process based on the meta, such as partition prune. + @SerializedName(value = "externalTableGetTableMetaTime") + private long externalTableGetTableMetaTime = 0; + // Total time to get partition values, including time to get partition values from external catalog and time to do + // some process based on the partition values, such as partition prune. + @SerializedName(value = "externalTableGetPartitionValuesTime") + private long externalTableGetPartitionValuesTime = 0; + // Total time to get partitions, including time to get partitions from external catalog and time to do some + // process based on the partitions, such as partition prune. + @SerializedName(value = "externalTableGetPartitionsTime") + private long externalTableGetPartitionsTime = 0; + // Total time to get partition files, including time to get partition files from external catalog and time to do + // some process based on the partition files, such as creating scan range. + @SerializedName(value = "externalTableGetPartitionFilesTime") + private long externalTableGetPartitionFilesTime = 0; + // Total time to get file scan tasks, including time to get file scan tasks from external catalog and time to do + // some process based on the file scan tasks, such as creating scan range. + @SerializedName(value = "externalTableGetFileScanTasksTime") + private long externalTableGetFileScanTasksTime = 0; @SerializedName(value = "externalTvfInitTime") private long externalTvfInitTime = 0; @SerializedName(value = "nereidsPartitiionPruneTime") @@ -504,10 +536,19 @@ private void updateExecutionSummaryProfile() { getPrettyTime(finalizeScanNodeFinishTime, finalizeScanNodeStartTime, TUnit.TIME_MS)); executionSummaryProfile.addInfoString(GET_SPLITS_TIME, getPrettyTime(getSplitsFinishTime, getSplitsStartTime, TUnit.TIME_MS)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_META_TIME, + getPrettyAccumulatedTime(externalCatalogMetaTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_TABLE_META_TIME, + getPrettyAccumulatedTime(externalTableGetTableMetaTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, + getPrettyAccumulatedTime(externalTableGetPartitionValuesTime)); executionSummaryProfile.addInfoString(GET_PARTITIONS_TIME, - getPrettyTime(getPartitionsFinishTime, getSplitsStartTime, TUnit.TIME_MS)); + getExternalTableMetaTime(externalTableGetPartitionsTime, getPartitionsFinishTime, getSplitsStartTime)); executionSummaryProfile.addInfoString(GET_PARTITION_FILES_TIME, - getPrettyTime(getPartitionFilesFinishTime, getPartitionsFinishTime, TUnit.TIME_MS)); + getExternalTableMetaTime(externalTableGetPartitionFilesTime, + getPartitionFilesFinishTime, getPartitionsFinishTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, + getPrettyAccumulatedTime(externalTableGetFileScanTasksTime)); executionSummaryProfile.addInfoString(SINK_SET_PARTITION_VALUES_TIME, getPrettyTime(sinkSetPartitionValuesFinishTime, sinkSetPartitionValuesStartTime, TUnit.TIME_MS)); executionSummaryProfile.addInfoString(CREATE_SCAN_RANGE_TIME, @@ -1062,6 +1103,20 @@ private String getPrettyTime(long end, long start, TUnit unit) { return RuntimeProfile.printCounter(end - start, unit); } + private String getPrettyAccumulatedTime(long timeMs) { + if (timeMs <= 0) { + return "N/A"; + } + return RuntimeProfile.printCounter(timeMs, TUnit.TIME_MS); + } + + private String getExternalTableMetaTime(long accumulatedTimeMs, long end, long start) { + if (accumulatedTimeMs > 0) { + return RuntimeProfile.printCounter(accumulatedTimeMs, TUnit.TIME_MS); + } + return getPrettyTime(end, start, TUnit.TIME_MS); + } + public void setTransactionBeginTime(TransactionType type) { this.transactionCommitBeginTime = TimeUtils.getStartTimeMs(); this.transactionType = type; @@ -1189,6 +1244,39 @@ public long getExternalCatalogMetaTimeMs() { return externalCatalogMetaTime; } + public synchronized void addExternalTableGetTableMetaTime(long ms) { + this.externalTableGetTableMetaTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionValuesTime(long ms) { + this.externalTableGetPartitionValuesTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionsTime(long ms) { + this.externalTableGetPartitionsTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionFilesTime(long ms) { + this.externalTableGetPartitionFilesTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetFileScanTasksTime(long ms) { + this.externalTableGetFileScanTasksTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalCatalogMetaTime(long ms) { + addExternalCatalogMetaTimeInternal(ms); + } + + private void addExternalCatalogMetaTimeInternal(long ms) { + this.externalCatalogMetaTime += ms; + } + public void addExternalTvfInitTime(long ms) { this.externalTvfInitTime += ms; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 3bd819bea62a16..8df635e5ea5f96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -34,6 +34,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.source.HiveSplit; @@ -105,6 +106,7 @@ public abstract class FileQueryScanNode extends FileScanNode { protected TableScanParams scanParams; protected FileSplitter fileSplitter; + protected SummaryProfile summaryProfile; // The data cache function only works for queries on Hive, Iceberg, Hudi(via HMS), and Paimon tables. // See: https://doris.incubator.apache.org/docs/dev/lakehouse/data-cache @@ -136,11 +138,12 @@ public FileQueryScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeNam public void init() throws UserException { super.init(); if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeStartTime(); + summaryProfile = ConnectContext.get().getExecutor().getSummaryProfile(); + summaryProfile.setInitScanNodeStartTime(); } doInitialize(); if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeFinishTime(); + summaryProfile.setInitScanNodeFinishTime(); } } @@ -161,6 +164,13 @@ protected void doInitialize() throws UserException { sessionVariable.maxInitialSplitNum); } + protected SummaryProfile getSummaryProfile() { + if (summaryProfile == null) { + summaryProfile = SummaryProfile.getSummaryProfile(ConnectContext.get()); + } + return summaryProfile; + } + // Init schema (Tuple/Slot) related params. protected void initSchemaParams() throws UserException { destSlotDescByName = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 05d78c37a649e5..03a37937562952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheKey; import org.apache.doris.datasource.SchemaCacheValue; @@ -237,7 +238,15 @@ public boolean isSupportedHmsTable() { protected synchronized void makeSureInitialized() { super.makeSureInitialized(); if (!objectCreated) { - remoteTable = loadHiveTable(); + long startTime = System.currentTimeMillis(); + try { + remoteTable = loadHiveTable(); + } finally { + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } + } if (remoteTable == null) { throw new IllegalArgumentException("Hms table not exists, table: " + getNameWithFullQualifiers()); } else { @@ -1290,11 +1299,18 @@ private Table loadHiveTable() { } public HiveExternalMetaCache.HivePartitionValues getHivePartitionValues(Optional snapshot) { + long startTime = System.currentTimeMillis(); HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .hive(getCatalog().getId()); try { List partitionColumnTypes = this.getPartitionColumnTypes(snapshot); - return cache.getPartitionValues(this, partitionColumnTypes); + HiveExternalMetaCache.HivePartitionValues partitionValues = cache.getPartitionValues(this, + partitionColumnTypes); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); + } + return partitionValues; } catch (Exception e) { if (e.getMessage().contains(HiveExternalMetaCache.ERR_CACHE_INCONSISTENCY)) { LOG.warn("Hive metastore cache inconsistency detected for table: {}.{}.{}. " @@ -1303,7 +1319,13 @@ public HiveExternalMetaCache.HivePartitionValues getHivePartitionValues(Optional Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableByEngine( getCatalog().getId(), getMetaCacheEngine(), getDbName(), getName()); List partitionColumnTypes = this.getPartitionColumnTypes(snapshot); - return cache.getPartitionValues(this, partitionColumnTypes); + HiveExternalMetaCache.HivePartitionValues partitionValues = cache.getPartitionValues(this, + partitionColumnTypes); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); + } + return partitionValues; } else { throw e; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index ad7539af255150..6c96dab3aa0ea2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -150,41 +150,51 @@ static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { } protected List getPartitions() throws AnalysisException { + long startTime = System.currentTimeMillis(); List resPartitions = Lists.newArrayList(); - HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .hive(hmsTable.getCatalog().getId()); - List partitionColumnTypes = hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable)); - if (!partitionColumnTypes.isEmpty()) { - // partitioned table - Collection partitionItems; - // partitions has benn pruned by Nereids, in PruneFileScanPartition, - // so just use the selected partitions. - this.totalPartitionNum = selectedPartitions.totalPartitionNum; - partitionItems = selectedPartitions.selectedPartitions.values(); - Preconditions.checkNotNull(partitionItems); - this.selectedPartitionNum = partitionItems.size(); - - // get partitions from cache - List> partitionValuesList = Lists.newArrayListWithCapacity(partitionItems.size()); - for (PartitionItem item : partitionItems) { - partitionValuesList.add( - ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); + try { + HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .hive(hmsTable.getCatalog().getId()); + List partitionColumnTypes = + hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable)); + if (!partitionColumnTypes.isEmpty()) { + // partitioned table + Collection partitionItems; + // partitions has benn pruned by Nereids, in PruneFileScanPartition, + // so just use the selected partitions. + this.totalPartitionNum = selectedPartitions.totalPartitionNum; + partitionItems = selectedPartitions.selectedPartitions.values(); + Preconditions.checkNotNull(partitionItems); + this.selectedPartitionNum = partitionItems.size(); + + // get partitions from cache + List> partitionValuesList = Lists.newArrayListWithCapacity(partitionItems.size()); + for (PartitionItem item : partitionItems) { + partitionValuesList.add( + ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); + } + resPartitions = cache.getAllPartitionsWithCache(hmsTable, partitionValuesList); + } else { + // non partitioned table, create a dummy partition to save location and inputformat, + // so that we can unify the interface. + HivePartition dummyPartition = new HivePartition(hmsTable.getOrBuildNameMapping(), true, + hmsTable.getRemoteTable().getSd().getInputFormat(), + hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); + this.totalPartitionNum = 1; + this.selectedPartitionNum = 1; + resPartitions.add(dummyPartition); } - resPartitions = cache.getAllPartitionsWithCache(hmsTable, partitionValuesList); - } else { - // non partitioned table, create a dummy partition to save location and inputformat, - // so that we can unify the interface. - HivePartition dummyPartition = new HivePartition(hmsTable.getOrBuildNameMapping(), true, - hmsTable.getRemoteTable().getSd().getInputFormat(), - hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); - this.totalPartitionNum = 1; - this.selectedPartitionNum = 1; - resPartitions.add(dummyPartition); - } - if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionsFinishTime(); + if (ConnectContext.get().getExecutor() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + getSummaryProfile().setGetPartitionsFinishTime(); + } + return resPartitions; + } catch (RuntimeException e) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + } + throw e; } - return resPartitions; } @Override @@ -228,6 +238,7 @@ public void startSplit(int numBackends) { Executor scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); String bindBrokerName = hmsTable.getCatalog().bindBrokerName(); AtomicInteger numFinishedPartitions = new AtomicInteger(0); + long startTime = System.currentTimeMillis(); CompletableFuture.runAsync(() -> { for (HivePartition partition : prunedPartitions) { if (batchException.get() != null || splitAssignment.isStop()) { @@ -255,6 +266,10 @@ public void startSplit(int numBackends) { splitAssignment.setException(batchException.get()); } if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionFilesTime( + System.currentTimeMillis() - startTime); + } splitAssignment.finishSchedule(); } } @@ -294,6 +309,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List allFiles, String bindBrokerName, int numBackends, boolean isBatchMode) throws IOException, UserException { List fileCaches; + long startTime = System.currentTimeMillis(); if (hiveTransaction != null) { try { fileCaches = getFileSplitByTransaction(cache, partitions, bindBrokerName); @@ -308,6 +324,9 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List 1, directoryLister, hmsTable); } + if (!isBatchMode && getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionFilesTime(System.currentTimeMillis() - startTime); + } long targetFileSplitSize = determineTargetFileSplitSize(fileCaches, isBatchMode); if (tableSample != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java index fbfe7a0a4daf3e..6c622e371c319e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -268,9 +269,14 @@ public static HudiMvccSnapshot getHudiMvccSnapshot(Optional table } public static long getLastTimeStamp(HMSExternalTable hmsTable) { + long startTime = System.currentTimeMillis(); HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); Option snapshotInstant = timeline.lastInstant(); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } if (!snapshotInstant.isPresent()) { return 0L; } @@ -279,40 +285,49 @@ public static long getLastTimeStamp(HMSExternalTable hmsTable) { public static TablePartitionValues getPartitionValues(Optional tableSnapshot, HMSExternalTable hmsTable) { - TablePartitionValues partitionValues = new TablePartitionValues(); - - HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); - HudiExternalMetaCache hudiExternalMetaCache = - Env.getCurrentEnv().getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()); - boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition(); - - if (tableSnapshot.isPresent()) { - if (tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION) { - // Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"; - return partitionValues; - } - String queryInstant = tableSnapshot.get().getValue().replaceAll("[-: ]", ""); - try { - partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> - hudiExternalMetaCache.getSnapshotPartitionValues(hmsTable, queryInstant, useHiveSyncPartition)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } - } else { - HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - return partitionValues; + long startTime = System.currentTimeMillis(); + try { + TablePartitionValues partitionValues = new TablePartitionValues(); + + HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); + HudiExternalMetaCache hudiExternalMetaCache = + Env.getCurrentEnv().getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()); + boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition(); + + if (tableSnapshot.isPresent()) { + if (tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION) { + // Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"; + return partitionValues; + } + String queryInstant = tableSnapshot.get().getValue().replaceAll("[-: ]", ""); + try { + partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> + hudiExternalMetaCache.getSnapshotPartitionValues( + hmsTable, queryInstant, useHiveSyncPartition)); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } else { + HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); + Option snapshotInstant = timeline.lastInstant(); + if (!snapshotInstant.isPresent()) { + return partitionValues; + } + try { + partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() + -> hudiExternalMetaCache.getPartitionValues(hmsTable, useHiveSyncPartition)); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } } - try { - partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> hudiExternalMetaCache.getPartitionValues(hmsTable, useHiveSyncPartition)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + return partitionValues; + } finally { + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); } } - return partitionValues; } public static HoodieTableMetaClient buildHudiTableMetaClient(String hudiBasePath, Configuration conf) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 75014c83ffd552..d5b454a8a5023c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -172,60 +172,67 @@ protected void doInitialize() throws UserException { initBackendPolicy(); initSchemaParams(); - hudiClient = hmsTable.getHudiClient(); - hudiClient.reloadActiveTimeline(); - basePath = hmsTable.getRemoteTable().getSd().getLocation(); - inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); - serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); - - if (scanParams != null && !scanParams.incrementalRead()) { - // Only support incremental read - throw new UserException("Not support function '" + scanParams.getParamType() + "' in hudi table"); - } - if (incrementalRead) { - if (isCowTable) { - try { - Map serd = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters(); - if ("true".equals(serd.get("hoodie.query.as.ro.table")) - && hmsTable.getRemoteTable().getTableName().endsWith("_ro")) { - // Incremental read RO table as RT table, I don't know why? - isCowTable = false; - LOG.warn("Execute incremental read on RO table: {}", hmsTable.getFullQualifiers()); + long tableMetaStartTime = System.currentTimeMillis(); + try { + hudiClient = hmsTable.getHudiClient(); + hudiClient.reloadActiveTimeline(); + basePath = hmsTable.getRemoteTable().getSd().getLocation(); + inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); + serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); + + if (scanParams != null && !scanParams.incrementalRead()) { + // Only support incremental read + throw new UserException("Not support function '" + scanParams.getParamType() + "' in hudi table"); + } + if (incrementalRead) { + if (isCowTable) { + try { + Map serd = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters(); + if ("true".equals(serd.get("hoodie.query.as.ro.table")) + && hmsTable.getRemoteTable().getTableName().endsWith("_ro")) { + // Incremental read RO table as RT table, I don't know why? + isCowTable = false; + LOG.warn("Execute incremental read on RO table: {}", hmsTable.getFullQualifiers()); + } + } catch (Exception e) { + // ignore } - } catch (Exception e) { - // ignore + } + if (incrementalRelation == null) { + throw new UserException("Failed to create incremental relation"); } } - if (incrementalRelation == null) { - throw new UserException("Failed to create incremental relation"); - } - } - timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); - TableSnapshot tableSnapshot = getQueryTableSnapshot(); - if (tableSnapshot != null) { - if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { - throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); + timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); + TableSnapshot tableSnapshot = getQueryTableSnapshot(); + if (tableSnapshot != null) { + if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { + throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); + } + queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", ""); + } else { + Option snapshotInstant = timeline.lastInstant(); + if (!snapshotInstant.isPresent()) { + prunedPartitions = Collections.emptyList(); + partitionInit = true; + return; + } + queryInstant = snapshotInstant.get().requestedTime(); } - queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", ""); - } else { - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - prunedPartitions = Collections.emptyList(); - partitionInit = true; - return; + + HudiSchemaCacheValue hudiSchemaCacheValue = HudiUtils.getSchemaCacheValue(hmsTable, queryInstant); + columnNames = hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList()); + columnTypes = hudiSchemaCacheValue.getColTypes(); + + fsView = Env.getCurrentEnv() + .getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()) + .getFsView(hmsTable.getOrBuildNameMapping()); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - tableMetaStartTime); } - queryInstant = snapshotInstant.get().requestedTime(); } - - HudiSchemaCacheValue hudiSchemaCacheValue = HudiUtils.getSchemaCacheValue(hmsTable, queryInstant); - columnNames = hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList()); - columnTypes = hudiSchemaCacheValue.getColTypes(); - - fsView = Env.getCurrentEnv() - .getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .getFsView(hmsTable.getOrBuildNameMapping()); // Todo: Get the current schema id of the table, instead of using -1. // In Be Parquet/Rrc reader, if `current table schema id == current file schema id`, then its // `table_info_node_ptr` will be `TableSchemaChangeHelper::ConstNode`. When using `ConstNode`, @@ -370,21 +377,30 @@ private List getPrunedPartitions(HoodieTableMetaClient metaClient } private List getIncrementalSplits() { + long startTime = System.currentTimeMillis(); if (canUseNativeReader()) { List splits = incrementalRelation.collectSplits(); noLogsSplitNum.addAndGet(splits.size()); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } return splits; } Option partitionColumns = hudiClient.getTableConfig().getPartitionFields(); List partitionNames = partitionColumns.isPresent() ? Arrays.asList(partitionColumns.get()) : Collections.emptyList(); - return incrementalRelation.collectFileSlices().stream().map(fileSlice -> generateHudiSplit(fileSlice, - HudiPartitionUtils.parsePartitionValues(partitionNames, fileSlice.getPartitionPath()), - incrementalRelation.getEndTs())).collect(Collectors.toList()); + List splits = incrementalRelation.collectFileSlices().stream() + .map(fileSlice -> generateHudiSplit(fileSlice, + HudiPartitionUtils.parsePartitionValues(partitionNames, fileSlice.getPartitionPath()), + incrementalRelation.getEndTs())) + .collect(Collectors.toList()); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } + return splits; } private void getPartitionSplits(HivePartition partition, List splits) throws IOException { - String partitionName; if (partition.isDummyPartition()) { partitionName = ""; @@ -424,6 +440,7 @@ private void getPartitionsSplits(List partitions, List spl Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); AtomicReference throwable = new AtomicReference<>(); + long startTime = System.currentTimeMillis(); partitions.forEach(partition -> executor.execute(() -> { try { getPartitionSplits(partition, splits); @@ -441,6 +458,9 @@ private void getPartitionsSplits(List partitions, List spl if (throwable.get() != null) { throw new RuntimeException(throwable.get().getMessage(), throwable.get()); } + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } @Override @@ -448,15 +468,7 @@ public List getSplits(int numBackends) throws UserException { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { return getIncrementalSplits(); } - if (!partitionInit) { - try { - prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> getPrunedPartitions(hudiClient)); - } catch (Exception e) { - throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); - } - partitionInit = true; - } + initPrunedPartitions(); List splits = Collections.synchronizedList(new ArrayList<>()); try { hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { @@ -469,6 +481,23 @@ public List getSplits(int numBackends) throws UserException { return splits; } + private void initPrunedPartitions() throws UserException { + if (partitionInit) { + return; + } + long startTime = System.currentTimeMillis(); + try { + prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() + -> getPrunedPartitions(hudiClient)); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + } + } catch (Exception e) { + throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); + } + partitionInit = true; + } + @Override public void startSplit(int numBackends) { if (prunedPartitions.isEmpty()) { @@ -477,6 +506,7 @@ public void startSplit(int numBackends) { } AtomicInteger numFinishedPartitions = new AtomicInteger(0); ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); + long startTime = System.currentTimeMillis(); CompletableFuture.runAsync(() -> { for (HivePartition partition : prunedPartitions) { if (batchException.get() != null || splitAssignment.isStop()) { @@ -506,6 +536,10 @@ public void startSplit(int numBackends) { splitAssignment.setException(batchException.get()); } if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime( + System.currentTimeMillis() - startTime); + } splitAssignment.finishSchedule(); } } @@ -522,15 +556,10 @@ public boolean isBatchMode() { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { return false; } - if (!partitionInit) { - // Non partition table will get one dummy partition - try { - prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> getPrunedPartitions(hudiClient)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } - partitionInit = true; + try { + initPrunedPartitions(); + } catch (UserException e) { + throw new RuntimeException(e.getMessage(), e); } int numPartitions = sessionVariable.getNumPartitionsInBatchMode(); return numPartitions >= 0 && prunedPartitions.size() >= numPartitions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java index c1626e915c65bb..65de2f9249df30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java @@ -37,22 +37,20 @@ public class IcebergApiSource implements IcebergSource { private final ExternalTable targetTable; - private final Table originTable; private final TupleDescriptor desc; + private Table originTable; public IcebergApiSource(ExternalTable table, TupleDescriptor desc, Map columnNameToRange) { + if (!(table instanceof IcebergExternalTable) && !(table instanceof IcebergSysExternalTable)) { + throw new IllegalArgumentException( + "Expected Iceberg table but got " + table.getClass().getSimpleName()); + } if (table instanceof IcebergExternalTable) { IcebergExternalTable icebergExtTable = (IcebergExternalTable) table; if (icebergExtTable.isView()) { throw new UnsupportedOperationException("IcebergApiSource does not support view"); } - this.originTable = IcebergUtils.getIcebergTable(icebergExtTable); - } else if (table instanceof IcebergSysExternalTable) { - this.originTable = ((IcebergSysExternalTable) table).getSysIcebergTable(); - } else { - throw new IllegalArgumentException( - "Expected Iceberg table but got " + table.getClass().getSimpleName()); } this.targetTable = table; this.desc = desc; @@ -64,12 +62,19 @@ public TupleDescriptor getDesc() { } @Override - public String getFileFormat() { - return IcebergUtils.getFileFormat(originTable).name(); + public String getFileFormat() throws MetaNotFoundException { + return IcebergUtils.getFileFormat(getIcebergTable()).name(); } @Override - public Table getIcebergTable() throws MetaNotFoundException { + public synchronized Table getIcebergTable() throws MetaNotFoundException { + if (originTable == null) { + if (targetTable instanceof IcebergExternalTable) { + originTable = IcebergUtils.getIcebergTable((IcebergExternalTable) targetTable); + } else { + originTable = ((IcebergSysExternalTable) targetTable).getSysIcebergTable(); + } + } return originTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java index a87b5408d5938b..cf33119cd60f11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java @@ -29,12 +29,11 @@ public class IcebergHMSSource implements IcebergSource { private final HMSExternalTable hmsTable; private final TupleDescriptor desc; - private final org.apache.iceberg.Table icebergTable; + private org.apache.iceberg.Table icebergTable; public IcebergHMSSource(HMSExternalTable hmsTable, TupleDescriptor desc) { this.hmsTable = hmsTable; this.desc = desc; - this.icebergTable = IcebergUtils.getIcebergTable(hmsTable); } @Override @@ -44,10 +43,13 @@ public TupleDescriptor getDesc() { @Override public String getFileFormat() throws DdlException, MetaNotFoundException { - return IcebergUtils.getFileFormat(icebergTable).name(); + return IcebergUtils.getFileFormat(getIcebergTable()).name(); } - public org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException { + public synchronized org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException { + if (icebergTable == null) { + icebergTable = IcebergUtils.getIcebergTable(hmsTable); + } return icebergTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 4e576a8bd67c9d..ad90cb8fb4af6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -212,24 +212,31 @@ public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckCol @Override protected void doInitialize() throws UserException { - icebergTable = source.getIcebergTable(); - partitionMapInfos = new HashMap<>(); - isPartitionedTable = icebergTable.spec().isPartitioned(); - // Metadata tables (system tables) are not BaseTable instances, so we need to handle this case - if (icebergTable instanceof BaseTable) { - formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); - } else { - // For metadata tables (e.g., snapshots, history), use a default format version - // These tables are always readable regardless of format version - formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION; - } - preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); - storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( - source.getCatalog().getCatalogProperty().getMetastoreProperties(), - source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), - icebergTable - ); - backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + long startTime = System.currentTimeMillis(); + try { + icebergTable = source.getIcebergTable(); + partitionMapInfos = new HashMap<>(); + isPartitionedTable = icebergTable.spec().isPartitioned(); + // Metadata tables (system tables) are not BaseTable instances, so we need to handle this case + if (icebergTable instanceof BaseTable) { + formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + } else { + // For metadata tables (e.g., snapshots, history), use a default format version + // These tables are always readable regardless of format version + formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION; + } + preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); + storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + source.getCatalog().getCatalogProperty().getMetastoreProperties(), + source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), + icebergTable + ); + backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } + } super.doInitialize(); } @@ -515,15 +522,23 @@ public void doStartSplit() throws UserException { try { preExecutionAuthenticator.execute( () -> { - CloseableIterable fileScanTasks = planFileScanTask(scan); - taskRef.set(fileScanTasks); - - CloseableIterator iterator = fileScanTasks.iterator(); - while (splitAssignment.needMoreSplit() && iterator.hasNext()) { - try { - splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next()))); - } catch (UserException e) { - throw new RuntimeException(e); + long startTime = System.currentTimeMillis(); + try { + CloseableIterable fileScanTasks = planFileScanTask(scan); + taskRef.set(fileScanTasks); + CloseableIterator iterator = fileScanTasks.iterator(); + while (splitAssignment.needMoreSplit() && iterator.hasNext()) { + try { + splitAssignment.addToQueue( + Lists.newArrayList(createIcebergSplit(iterator.next()))); + } catch (UserException e) { + throw new RuntimeException(e); + } + } + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime( + System.currentTimeMillis() - startTime); } } } @@ -916,10 +931,12 @@ private List doGetSplits(int numBackends) throws UserException { // Normal table scan planning TableScan scan = createTableScan(); + long startTime = System.currentTimeMillis(); try (CloseableIterable fileScanTasks = planFileScanTask(scan)) { if (tableLevelPushDownCount) { int needSplitCnt = countFromSnapshot < COUNT_WITH_PARALLEL_SPLITS - ? 1 : sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()) * numBackends; + ? 1 : sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()) + * numBackends; for (FileScanTask next : fileScanTasks) { splits.add(createIcebergSplit(next)); if (splits.size() >= needSplitCnt) { @@ -935,6 +952,10 @@ private List doGetSplits(int numBackends) throws UserException { } } catch (IOException e) { throw new UserException(e.getMessage(), e.getCause()); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } selectedPartitionNum = partitionMapInfos.size(); @@ -945,10 +966,15 @@ private List doGetSplits(int numBackends) throws UserException { private List doGetSystemTableSplits() throws UserException { List splits = new ArrayList<>(); TableScan scan = createTableScan(); + long startTime = System.currentTimeMillis(); try (CloseableIterable fileScanTasks = scan.planFiles()) { fileScanTasks.forEach(task -> splits.add(createIcebergSysSplit(task))); } catch (IOException e) { throw new UserException(e.getMessage(), e); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } selectedPartitionNum = 0; return splits; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index a8e331f9e468f6..1e9ac223df0c8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -175,6 +175,7 @@ public PaimonScanNode(PlanNodeId id, @Override protected void doInitialize() throws UserException { super.doInitialize(); + long startTime = System.currentTimeMillis(); source = new PaimonSource(desc); serializedTable = PaimonUtil.encodeObjectToString(source.getPaimonTable()); // Todo: Get the current schema id of the table, instead of using -1. @@ -187,6 +188,9 @@ protected void doInitialize() throws UserException { ); backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); backendPaimonOptions = getBackendPaimonOptions(); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } } @VisibleForTesting @@ -612,26 +616,33 @@ public Map getIncrReadParams() throws UserException { @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { - Table paimonTable = getProcessedTable(); - List fieldNames = paimonTable.rowType().getFieldNames(); - int[] projected = desc.getSlots().stream().mapToInt( - slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) - .filter(i -> i >= 0) - .toArray(); - ReadBuilder readBuilder = paimonTable.newReadBuilder(); - TableScan scan = readBuilder.withFilter(predicates) - .withProjection(projected) - .newScan(); - PaimonMetricRegistry registry = new PaimonMetricRegistry(); - if (scan instanceof InnerTableScan) { - scan = ((InnerTableScan) scan).withMetricRegistry(registry); - } - List splits = scan.plan().splits(); - PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); - if (!registry.getAllGroups().isEmpty()) { - registry.clear(); + long startTime = System.currentTimeMillis(); + try { + Table paimonTable = getProcessedTable(); + List fieldNames = paimonTable.rowType().getFieldNames(); + int[] projected = desc.getSlots().stream().mapToInt( + slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) + .filter(i -> i >= 0) + .toArray(); + ReadBuilder readBuilder = paimonTable.newReadBuilder(); + TableScan scan = readBuilder.withFilter(predicates) + .withProjection(projected) + .newScan(); + PaimonMetricRegistry registry = new PaimonMetricRegistry(); + if (scan instanceof InnerTableScan) { + scan = ((InnerTableScan) scan).withMetricRegistry(registry); + } + List splits = scan.plan().splits(); + PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); + if (!registry.getAllGroups().isEmpty()) { + registry.clear(); + } + return splits; + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } - return splits; } @VisibleForTesting diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java index ef6fd3d63f3ce4..afe40a68dc8c16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java @@ -77,4 +77,28 @@ public void testPreloadExternalMetadataTimeCounter() { Assertions.assertEquals(20, profile.getNereidsPreloadExternalMetadataTimeMs()); Assertions.assertEquals("20ms", profile.getPrettyNereidsPreloadExternalMetadataTime()); } + + @Test + public void testExternalTableMetaSummary() { + SummaryProfile profile = new SummaryProfile(); + profile.addExternalTableGetTableMetaTime(2); + profile.addExternalTableGetPartitionValuesTime(3); + profile.addExternalTableGetPartitionsTime(5); + profile.addExternalTableGetPartitionFilesTime(7); + profile.addExternalTableGetFileScanTasksTime(11); + + profile.update(ImmutableMap.of()); + + RuntimeProfile executionSummary = profile.getExecutionSummary(); + Assertions.assertEquals("28ms", executionSummary.getInfoString(SummaryProfile.EXTERNAL_TABLE_META_TIME)); + Assertions.assertEquals("2ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_TABLE_META_TIME)); + Assertions.assertEquals("3ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME)); + Assertions.assertEquals("5ms", executionSummary.getInfoString(SummaryProfile.GET_PARTITIONS_TIME)); + Assertions.assertEquals("7ms", executionSummary.getInfoString(SummaryProfile.GET_PARTITION_FILES_TIME)); + Assertions.assertEquals("11ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME)); + Assertions.assertEquals(28, profile.getExternalCatalogMetaTimeMs()); + } } From 12e40822272e14a96bffb6630671abfe7111bd7a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 3 Jun 2026 10:22:02 +0800 Subject: [PATCH 02/24] [fix](be) Use shared IOContext in file scanner readers (#64033) Problem Summary: FileScanner kept passing raw IOContext pointers to several file readers, so DelegateReader could still create a shallow-copied IOContext on the hot scan path. That left different IOContext instances inside the same reader stack and could also dereference missing child stats pointers when an IOContext existed without file reader stats. This change keeps FileScanner's IOContext in a shared holder, passes it through CSV, text, JSON, native, Parquet, ORC, and table-format reader variants, and makes Native/Parquet/ORC use the shared DelegateReader API when a holder is available. Tracing/stat updates now check the nested stats pointer before use. --- be/src/exec/scan/file_scanner.cpp | 23 +++++++++---------- be/src/exec/scan/file_scanner.h | 5 +++-- be/src/format/arrow/arrow_stream_reader.cpp | 3 ++- be/src/format/csv/csv_reader.cpp | 3 ++- be/src/format/json/new_json_reader.cpp | 3 ++- be/src/format/native/native_reader.cpp | 25 +++++++++++++++++---- be/src/format/native/native_reader.h | 6 +++++ be/src/format/orc/vorc_reader.cpp | 7 +++--- be/src/format/orc/vorc_reader.h | 7 +++--- be/src/format/parquet/vparquet_reader.cpp | 21 ++++++++++++----- be/src/format/text/text_reader.cpp | 6 +++-- be/src/format/text/text_reader.h | 3 ++- 12 files changed, 76 insertions(+), 36 deletions(-) diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 2117bc3873b6ff..188c1f6efdb6ef 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1151,7 +1151,7 @@ Status FileScanner::_get_next_reader() { : nullptr; std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, _state->query_options().batch_size, - const_cast(&_state->timezone_obj()), _io_ctx.get(), _state, + const_cast(&_state->timezone_obj()), _io_ctx, _state, file_meta_cache_ptr, _state->query_options().enable_parquet_lazy_mat); if (_row_id_column_iterator_pair.second != -1) { @@ -1176,7 +1176,7 @@ Status FileScanner::_get_next_reader() { : nullptr; std::unique_ptr orc_reader = OrcReader::create_unique( _profile, _state, *_params, range, _state->query_options().batch_size, - _state->timezone(), _io_ctx.get(), file_meta_cache_ptr, + _state->timezone(), _io_ctx, file_meta_cache_ptr, _state->query_options().enable_orc_lazy_mat); if (_row_id_column_iterator_pair.second != -1) { RETURN_IF_ERROR(_create_row_id_column_iterator()); @@ -1201,17 +1201,17 @@ Status FileScanner::_get_next_reader() { case TFileFormatType::FORMAT_CSV_DEFLATE: case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: case TFileFormatType::FORMAT_PROTO: { - auto reader = - CsvReader::create_unique(_state, _profile, &_counter, *_params, range, - _file_slot_descs, _state->batch_size(), _io_ctx.get()); + auto reader = CsvReader::create_unique(_state, _profile, &_counter, *_params, range, + _file_slot_descs, _state->batch_size(), nullptr, + _io_ctx); init_status = reader->init_reader(_is_load); _cur_reader = std::move(reader); break; } case TFileFormatType::FORMAT_TEXT: { auto reader = TextReader::create_unique(_state, _profile, &_counter, *_params, range, - _file_slot_descs, _state->batch_size(), - _io_ctx.get()); + _file_slot_descs, _state->batch_size(), nullptr, + _io_ctx); init_status = reader->init_reader(_is_load); _cur_reader = std::move(reader); break; @@ -1219,7 +1219,7 @@ Status FileScanner::_get_next_reader() { case TFileFormatType::FORMAT_JSON: { _cur_reader = NewJsonReader::create_unique(_state, _profile, &_counter, *_params, range, _file_slot_descs, &_scanner_eof, - _state->batch_size(), _io_ctx.get()); + _state->batch_size(), nullptr, _io_ctx); init_status = ((NewJsonReader*)(_cur_reader.get())) ->init_reader(_col_default_value_ctx, _is_load); break; @@ -1241,8 +1241,7 @@ Status FileScanner::_get_next_reader() { break; } case TFileFormatType::FORMAT_NATIVE: { - auto reader = - NativeReader::create_unique(_profile, *_params, range, _io_ctx.get(), _state); + auto reader = NativeReader::create_unique(_profile, *_params, range, _io_ctx, _state); init_status = reader->init_reader(); _cur_reader = std::move(reader); need_to_get_parsed_schema = false; @@ -1681,7 +1680,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, case TFileFormatType::FORMAT_PARQUET: { std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, 1, - const_cast(&_state->timezone_obj()), _io_ctx.get(), + const_cast(&_state->timezone_obj()), _io_ctx, _state, file_meta_cache_ptr, false); RETURN_IF_ERROR(parquet_reader->read_by_rows(row_ids)); @@ -1691,7 +1690,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, } case TFileFormatType::FORMAT_ORC: { std::unique_ptr orc_reader = OrcReader::create_unique( - _profile, _state, *_params, range, 1, _state->timezone(), _io_ctx.get(), + _profile, _state, *_params, range, 1, _state->timezone(), _io_ctx, file_meta_cache_ptr, false); RETURN_IF_ERROR(orc_reader->read_by_rows(row_ids)); diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 88bf97df142403..b4c0c172100fa4 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -200,7 +200,8 @@ class FileScanner : public Scanner { std::unique_ptr _file_cache_statistics; std::unique_ptr _file_reader_stats; - std::unique_ptr _io_ctx; + // Reader stacks retain this context so delegate readers never outlive their scan state. + std::shared_ptr _io_ctx; // Whether to fill partition columns from path, default is true. bool _fill_partition_from_path = true; @@ -294,7 +295,7 @@ class FileScanner : public Scanner { }; Status _init_io_ctx() { - _io_ctx.reset(new io::IOContext()); + _io_ctx = std::make_shared(); _io_ctx->query_id = &_state->query_id(); return Status::OK(); }; diff --git a/be/src/format/arrow/arrow_stream_reader.cpp b/be/src/format/arrow/arrow_stream_reader.cpp index 5fa9143dab2ac8..26d8aacd350fa8 100644 --- a/be/src/format/arrow/arrow_stream_reader.cpp +++ b/be/src/format/arrow/arrow_stream_reader.cpp @@ -58,7 +58,8 @@ ArrowStreamReader::~ArrowStreamReader() = default; Status ArrowStreamReader::init_reader() { io::FileReaderSPtr file_reader; RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &file_reader, _state, false)); - _file_reader = _io_ctx ? std::make_shared(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; _pip_stream = ArrowPipInputStream::create_unique(_file_reader); diff --git a/be/src/format/csv/csv_reader.cpp b/be/src/format/csv/csv_reader.cpp index 7bc340e7f21774..d6bee41acdcf91 100644 --- a/be/src/format/csv/csv_reader.cpp +++ b/be/src/format/csv/csv_reader.cpp @@ -606,7 +606,8 @@ Status CsvReader::_create_file_reader(bool need_schema) { io::DelegateReader::AccessMode::SEQUENTIAL, _io_ctx, io::PrefetchRange(_range.start_offset, _range.start_offset + _range.size))); } - _file_reader = _io_ctx ? std::make_shared(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; } diff --git a/be/src/format/json/new_json_reader.cpp b/be/src/format/json/new_json_reader.cpp index 7b9bde931b4cde..31cb50f409698d 100644 --- a/be/src/format/json/new_json_reader.cpp +++ b/be/src/format/json/new_json_reader.cpp @@ -471,7 +471,8 @@ Status NewJsonReader::_open_file_reader(bool need_schema) { io::DelegateReader::AccessMode::SEQUENTIAL, _io_ctx, io::PrefetchRange(_range.start_offset, _range.size))); } - _file_reader = _io_ctx ? std::make_shared(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; } diff --git a/be/src/format/native/native_reader.cpp b/be/src/format/native/native_reader.cpp index 0cb0c1ca72a6c0..408e71213e089c 100644 --- a/be/src/format/native/native_reader.cpp +++ b/be/src/format/native/native_reader.cpp @@ -19,6 +19,8 @@ #include +#include + #include "core/block/block.h" #include "format/native/native_format.h" #include "io/file_factory.h" @@ -40,6 +42,16 @@ NativeReader::NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& _io_ctx(io_ctx), _state(state) {} +NativeReader::NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, + std::shared_ptr io_ctx_holder, RuntimeState* state) + : _profile(profile), + _scan_params(params), + _scan_range(range), + _io_ctx(io_ctx_holder ? io_ctx_holder.get() : nullptr), + _io_ctx_holder(std::move(io_ctx_holder)), + _state(state) {} + NativeReader::~NativeReader() { (void)close(); } @@ -129,15 +141,20 @@ Status NativeReader::init_reader() { io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state, file_description); - auto reader_res = io::DelegateReader::create_file_reader( - _profile, system_properties, file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx); + auto reader_res = + _io_ctx_holder ? io::DelegateReader::create_file_reader( + _profile, system_properties, file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, + std::static_pointer_cast(_io_ctx_holder)) + : io::DelegateReader::create_file_reader( + _profile, system_properties, file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx); if (!reader_res.has_value()) { return reader_res.error(); } _file_reader = reader_res.value(); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _file_reader = std::make_shared(_file_reader, _io_ctx->file_reader_stats); } diff --git a/be/src/format/native/native_reader.h b/be/src/format/native/native_reader.h index 65d70816628eea..d72544ea472c08 100644 --- a/be/src/format/native/native_reader.h +++ b/be/src/format/native/native_reader.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -53,6 +54,10 @@ class NativeReader : public GenericReader { NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, io::IOContext* io_ctx, RuntimeState* state); + NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, std::shared_ptr io_ctx_holder, + RuntimeState* state); + ~NativeReader() override; // Initialize underlying file reader and any format specific state. @@ -82,6 +87,7 @@ class NativeReader : public GenericReader { io::FileReaderSPtr _file_reader; io::IOContext* _io_ctx = nullptr; + std::shared_ptr _io_ctx_holder; RuntimeState* _state = nullptr; bool _eof = false; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 8b42d2cceb3057..d2224ff2ccec5d 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2481,7 +2481,7 @@ Status OrcReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { _reader_metrics.SelectedRowGroupCount); COUNTER_UPDATE(_orc_profile.evaluated_row_group_count, _reader_metrics.EvaluatedRowGroupCount); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _io_ctx->file_reader_stats->read_rows += _reader_metrics.ReadRowCount; } } @@ -3492,7 +3492,7 @@ void ORCFileInputStream::_build_small_ranges_input_stripe_streams( std::make_shared(_profile, _file_reader, merged_range); std::shared_ptr tracing_file_reader; - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { tracing_file_reader = std::make_shared( std::move(merge_range_file_reader), _io_ctx->file_reader_stats); } else { @@ -3525,7 +3525,8 @@ void ORCFileInputStream::_build_large_ranges_input_stripe_streams( for (const auto& range : ranges) { auto stripe_stream_input_stream = std::make_shared( getName(), - _io_ctx ? std::make_shared(_file_reader, + _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(_file_reader, _io_ctx->file_reader_stats) : _file_reader, _io_ctx, _profile); diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 7305e5637ff8dd..3f77d41f63990f 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -836,9 +836,10 @@ class ORCFileInputStream : public orc::InputStream, public ProfileCollector { : _file_name(file_name), _inner_reader(inner_reader), _file_reader(inner_reader), - _tracing_file_reader(io_ctx ? std::make_shared( - _file_reader, io_ctx->file_reader_stats) - : _file_reader), + _tracing_file_reader(io_ctx && io_ctx->file_reader_stats + ? std::make_shared( + _file_reader, io_ctx->file_reader_stats) + : _file_reader), _orc_once_max_read_bytes(orc_once_max_read_bytes), _orc_max_merge_distance_bytes(orc_max_merge_distance_bytes), _io_ctx(io_ctx), diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index a1e376ce55ebfe..5fc1c92d06d0e7 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -324,10 +324,18 @@ Status ParquetReader::_open_file() { _scan_range.__isset.modification_time ? _scan_range.modification_time : 0; io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state, _file_description); - _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( - _profile, _system_properties, _file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx)); - _tracing_file_reader = _io_ctx ? std::make_shared( + if (_io_ctx_holder) { + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, _system_properties, _file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, + std::static_pointer_cast(_io_ctx_holder))); + } else { + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, _system_properties, _file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx)); + } + _tracing_file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared( _file_reader, _io_ctx->file_reader_stats) : _file_reader; } @@ -833,7 +841,7 @@ Status ParquetReader::_next_row_group_reader() { } _reader_statistics.read_rows += candidate_row_ranges.count(); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _io_ctx->file_reader_stats->read_rows += candidate_row_ranges.count(); } @@ -883,7 +891,8 @@ Status ParquetReader::_next_row_group_reader() { : _file_reader; } _current_group_reader.reset(new RowGroupReader( - _io_ctx ? std::make_shared(group_file_reader, + _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(group_file_reader, _io_ctx->file_reader_stats) : group_file_reader, _read_table_columns, _current_row_group_index.row_group_id, row_group, _ctz, _io_ctx, diff --git a/be/src/format/text/text_reader.cpp b/be/src/format/text/text_reader.cpp index 388cd56a0ddb90..bf06d915c8e9e1 100644 --- a/be/src/format/text/text_reader.cpp +++ b/be/src/format/text/text_reader.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/compiler_util.h" // IWYU pragma: keep @@ -114,8 +115,9 @@ void HiveTextFieldSplitter::_split_field_multi_char(const Slice& line, TextReader::TextReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, size_t batch_size, - io::IOContext* io_ctx) - : CsvReader(state, profile, counter, params, range, file_slot_descs, batch_size, io_ctx) {} + io::IOContext* io_ctx, std::shared_ptr io_ctx_holder) + : CsvReader(state, profile, counter, params, range, file_slot_descs, batch_size, io_ctx, + std::move(io_ctx_holder)) {} Status TextReader::_init_options() { // get column_separator and line_delimiter diff --git a/be/src/format/text/text_reader.h b/be/src/format/text/text_reader.h index 60b0fb2f8b544c..9667e05b19945d 100644 --- a/be/src/format/text/text_reader.h +++ b/be/src/format/text/text_reader.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -57,7 +58,7 @@ class TextReader : public CsvReader { TextReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, size_t batch_size, - io::IOContext* io_ctx); + io::IOContext* io_ctx, std::shared_ptr io_ctx_holder = nullptr); ~TextReader() override = default; From f0fda1ce2d1cbb397b24e70629ccfd561111143b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 10 Jun 2026 09:54:05 +0800 Subject: [PATCH 03/24] [fix](iceberg) Fix wrong processing for `NaN` (#64315) This change fixes Iceberg writes when FLOAT or DOUBLE partition values contain NaN or infinity. Previously, the BE Iceberg writer serialized floating-point partition values with std::to_string(), which produced lowercase special values such as "nan", "inf", and "-inf". These strings are not accepted by Java's Float.parseFloat() / Double.parseDouble() in the FE partition commit path, causing writes with NaN partition values to fail. The fix canonicalizes BE floating-point partition serialization to Java/Iceberg-compatible values: "NaN", "Infinity", and "-Infinity". It also makes FE partition value parsing tolerate legacy lowercase forms such as "nan" and "-inf". Unit tests were added for BE partition value serialization and FE parsing of special floating-point partition values. --- .../writer/iceberg/partition_transformers.cpp | 20 ++++++++++++++-- .../iceberg/partition_transformers_test.cpp | 24 +++++++++++++++++++ .../datasource/iceberg/IcebergUtils.java | 18 ++++++++++++-- .../datasource/iceberg/IcebergUtilsTest.java | 20 ++++++++++++++++ 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp index 8b49d68573c1e4..e38e0e4c5eb8a7 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp @@ -18,12 +18,28 @@ #include "exec/sink/writer/iceberg/partition_transformers.h" #include +#include #include "core/types.h" #include "format/table/iceberg/partition_spec.h" namespace doris { +namespace { + +template +std::string floating_point_partition_value_to_string(T value) { + if (std::isnan(value)) { + return "NaN"; + } + if (std::isinf(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + return std::to_string(value); +} + +} // namespace + const std::chrono::sys_days PartitionColumnTransformUtils::EPOCH = std::chrono::sys_days( std::chrono::year {1970} / std::chrono::January / std::chrono::day {1}); @@ -225,10 +241,10 @@ std::string PartitionColumnTransform::get_partition_value(const DataTypePtr type return std::to_string(std::any_cast(value)); } case TYPE_FLOAT: { - return std::to_string(std::any_cast(value)); + return floating_point_partition_value_to_string(std::any_cast(value)); } case TYPE_DOUBLE: { - return std::to_string(std::any_cast(value)); + return floating_point_partition_value_to_string(std::any_cast(value)); } case TYPE_VARCHAR: case TYPE_CHAR: diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp index 69758b794dbd5a..94b8aec8c77ea6 100644 --- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp +++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp @@ -19,6 +19,8 @@ #include +#include + #include "core/data_type/data_type_date_or_datetime_v2.h" namespace doris { @@ -115,6 +117,28 @@ TEST_F(PartitionTransformersTest, test_string_truncate_transform) { } } +TEST_F(PartitionTransformersTest, test_floating_point_special_partition_value) { + auto float_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_FLOAT, false); + auto double_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DOUBLE, false); + IdentityPartitionColumnTransform float_transform(float_type); + IdentityPartitionColumnTransform double_transform(double_type); + + EXPECT_EQ("NaN", float_transform.get_partition_value( + float_type, std::numeric_limits::quiet_NaN())); + EXPECT_EQ("Infinity", float_transform.get_partition_value( + float_type, std::numeric_limits::infinity())); + EXPECT_EQ("-Infinity", float_transform.get_partition_value( + float_type, -std::numeric_limits::infinity())); + EXPECT_EQ("NaN", double_transform.get_partition_value( + double_type, std::numeric_limits::quiet_NaN())); + EXPECT_EQ("Infinity", double_transform.get_partition_value( + double_type, std::numeric_limits::infinity())); + EXPECT_EQ("-Infinity", double_transform.get_partition_value( + double_type, -std::numeric_limits::infinity())); +} + TEST_F(PartitionTransformersTest, test_integer_bucket_transform) { const std::vector values({34, -123}); // 2017239379, -471378254 auto column = ColumnInt32::create(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index b8b8d5967491df..8db319a7e2497a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -982,9 +982,9 @@ public static Object parsePartitionValueFromString(String valueStr, org.apache.i case LONG: return Long.parseLong(valueStr); case FLOAT: - return Float.parseFloat(valueStr); + return Float.parseFloat(normalizeFloatingPointPartitionValue(valueStr)); case DOUBLE: - return Double.parseDouble(valueStr); + return Double.parseDouble(normalizeFloatingPointPartitionValue(valueStr)); case BOOLEAN: return Boolean.parseBoolean(valueStr); case DATE: @@ -1003,6 +1003,20 @@ public static Object parsePartitionValueFromString(String valueStr, org.apache.i } } + private static String normalizeFloatingPointPartitionValue(String valueStr) { + if ("nan".equalsIgnoreCase(valueStr)) { + return "NaN"; + } + if ("inf".equalsIgnoreCase(valueStr) || "+inf".equalsIgnoreCase(valueStr) + || "infinity".equalsIgnoreCase(valueStr) || "+infinity".equalsIgnoreCase(valueStr)) { + return "Infinity"; + } + if ("-inf".equalsIgnoreCase(valueStr) || "-infinity".equalsIgnoreCase(valueStr)) { + return "-Infinity"; + } + return valueStr; + } + /** * Parse timestamp string to microseconds using Doris's built-in datetime * parser. diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 47011cafc9473c..112873433b03ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -373,6 +373,26 @@ public void testGetIdentityPartitionInfoMapSupportsFloatingPointPartitions() { Double.doubleToLongBits(Double.parseDouble(serializedDouble))); } + @Test + public void testParseFloatingPointPartitionValueSupportsSpecialValues() { + Assert.assertTrue(Float.isNaN( + (Float) IcebergUtils.parsePartitionValueFromString("NaN", Types.FloatType.get()))); + Assert.assertTrue(Float.isNaN( + (Float) IcebergUtils.parsePartitionValueFromString("nan", Types.FloatType.get()))); + Assert.assertEquals(Float.POSITIVE_INFINITY, + (Float) IcebergUtils.parsePartitionValueFromString("Infinity", Types.FloatType.get()), 0.0F); + Assert.assertEquals(Float.NEGATIVE_INFINITY, + (Float) IcebergUtils.parsePartitionValueFromString("-inf", Types.FloatType.get()), 0.0F); + Assert.assertTrue(Double.isNaN( + (Double) IcebergUtils.parsePartitionValueFromString("NaN", Types.DoubleType.get()))); + Assert.assertTrue(Double.isNaN( + (Double) IcebergUtils.parsePartitionValueFromString("nan", Types.DoubleType.get()))); + Assert.assertEquals(Double.POSITIVE_INFINITY, + (Double) IcebergUtils.parsePartitionValueFromString("Infinity", Types.DoubleType.get()), 0.0D); + Assert.assertEquals(Double.NEGATIVE_INFINITY, + (Double) IcebergUtils.parsePartitionValueFromString("-inf", Types.DoubleType.get()), 0.0D); + } + @Test public void testGetMatchingManifest() { From 19fcc68c644f9083359abe3a8737b173aa3d99c1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 29 Jun 2026 10:33:26 +0800 Subject: [PATCH 04/24] [fix](fe) Reject Iceberg v3 row lineage columns (#63825) Problem Summary: Creating an Iceberg table with format-version 3 could accept user columns named _row_id or _last_updated_sequence_number. Doris later appends Iceberg row lineage hidden columns with the same names, which can produce duplicate hidden-column metadata and confusing insert/query behavior. This change validates Iceberg create table definitions, including CTAS, and rejects those reserved row lineage column names when the requested Iceberg format version is 3 or newer. --- .../iceberg/IcebergMetadataOps.java | 8 ++- .../datasource/iceberg/IcebergUtils.java | 27 +++++++ .../plans/commands/info/CreateTableInfo.java | 36 ++++++++++ .../iceberg/IcebergDDLAndDMLPlanTest.java | 71 +++++++++++++++++++ .../iceberg/IcebergMetadataOpTest.java | 55 ++++++++++++++ 5 files changed, 196 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index db98bb283d41ee..04fc1c2865c82b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -361,10 +361,16 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx Schema schema = new Schema(visit.asNestedType().asStructType().fields()); Map properties = createTableInfo.getProperties(); properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); - properties.putIfAbsent(TableProperties.FORMAT_VERSION, "2"); + Map catalogProperties = dorisCatalog.getProperties(); + if (!properties.containsKey(TableProperties.FORMAT_VERSION) + && !IcebergUtils.hasIcebergCatalogFormatVersion(catalogProperties)) { + properties.put(TableProperties.FORMAT_VERSION, "2"); + } properties.putIfAbsent(TableProperties.DELETE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); properties.putIfAbsent(TableProperties.UPDATE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); properties.putIfAbsent(TableProperties.MERGE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); + createTableInfo.validateIcebergRowLineageColumns( + IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalogProperties)); PartitionSpec partitionSpec = IcebergUtils.solveIcebergPartitionSpec(createTableInfo.getPartitionDesc(), schema); // Build and create table with optional sort order diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 8db319a7e2497a..a78bdc57c7afb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -190,6 +190,33 @@ public Integer initialValue() { private static final Pattern SNAPSHOT_ID = Pattern.compile("\\d+"); + public static boolean hasIcebergCatalogFormatVersion(Map catalogProperties) { + return catalogProperties.containsKey(CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.FORMAT_VERSION) + || catalogProperties.containsKey(CatalogProperties.TABLE_DEFAULT_PREFIX + + TableProperties.FORMAT_VERSION); + } + + public static int getEffectiveIcebergFormatVersion(Map tableProperties, + Map catalogProperties) { + String formatVersion = catalogProperties.get(CatalogProperties.TABLE_OVERRIDE_PREFIX + + TableProperties.FORMAT_VERSION); + if (formatVersion == null) { + formatVersion = tableProperties.get(TableProperties.FORMAT_VERSION); + if (formatVersion == null) { + formatVersion = catalogProperties.get(CatalogProperties.TABLE_DEFAULT_PREFIX + + TableProperties.FORMAT_VERSION); + } + } + if (formatVersion == null) { + return 2; + } + try { + return Integer.parseInt(formatVersion); + } catch (NumberFormatException ignored) { + return 2; + } + } + public static Expression convertToIcebergExpr(Expr expr, Schema schema) { if (expr == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index df88e84f5e6146..35d87e65279b59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.info.TableNameInfo; @@ -378,6 +379,9 @@ private void checkEngineWithCatalog() { + " Make sure 'engine' type is specified when use the catalog: " + ctlName); } } + if (Strings.isNullOrEmpty(ctlName)) { + return; + } CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); if (catalog instanceof HMSExternalCatalog && !engineName.equals(ENGINE_HIVE)) { throw new AnalysisException("Hms type catalog can only use `hive` engine."); @@ -788,6 +792,10 @@ public void validate(ConnectContext ctx) { + "and you can use 'bucket(num, column)' in 'PARTITIONED BY'."); } + if (engineName.equalsIgnoreCase(ENGINE_ICEBERG)) { + validateIcebergRowLineageColumns(); + } + // Validate Iceberg sort order columns if (sortOrderFields != null && !sortOrderFields.isEmpty()) { if (!engineName.equalsIgnoreCase(ENGINE_ICEBERG)) { @@ -1090,6 +1098,34 @@ private void validateKeyColumns() { } } + /** + * Validate that Iceberg v3 tables do not define row lineage reserved columns. + */ + public void validateIcebergRowLineageColumns(int formatVersion) { + if (formatVersion < IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) { + return; + } + for (ColumnDefinition columnDef : columns) { + if (IcebergUtils.isIcebergRowLineageColumn(columnDef.getName())) { + throw new AnalysisException("Cannot create Iceberg v" + formatVersion + + " table with reserved row lineage column: " + columnDef.getName()); + } + } + } + + private void validateIcebergRowLineageColumns() { + validateIcebergRowLineageColumns(getEffectiveIcebergFormatVersion()); + } + + private int getEffectiveIcebergFormatVersion() { + CatalogIf catalog = Strings.isNullOrEmpty(ctlName) ? null + : Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); + if (catalog instanceof IcebergExternalCatalog) { + return IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalog.getProperties()); + } + return IcebergUtils.getEffectiveIcebergFormatVersion(properties, Collections.emptyMap()); + } + /** * analyzeEngine */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 69506a505cb21e..0a798faff93fef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; @@ -277,6 +278,76 @@ public void testIcebergDeletePlanAddsRowIdProject() throws Exception { assertContainsPhysicalSink(physicalPlan, PhysicalIcebergDeleteSink.class); } + @Test + public void testCreateIcebergV3TableRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String rowIdSql = "create table " + rowIdTable + + " (_row_id bigint) properties('format-version'='3')"; + LogicalPlan rowIdPlan = parseStmt(rowIdSql); + Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext)); + + String lastUpdatedSequenceNumberTable = + "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String lastUpdatedSequenceNumberSql = "create table " + lastUpdatedSequenceNumberTable + + " (_last_updated_sequence_number bigint) properties('format-version'='3')"; + LogicalPlan lastUpdatedSequenceNumberPlan = parseStmt(lastUpdatedSequenceNumberSql); + Assertions.assertTrue(lastUpdatedSequenceNumberPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) lastUpdatedSequenceNumberPlan).getCreateTableInfo() + .validate(connectContext)); + + String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String formatV2Sql = "create table " + formatV2Table + + " (_row_id bigint) properties('format-version'='2')"; + LogicalPlan formatV2Plan = parseStmt(formatV2Sql); + Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand); + Assertions.assertDoesNotThrow( + () -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext)); + } + + @Test + public void testCreateIcebergDefaultV3TableRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + IcebergExternalCatalog catalog = (IcebergExternalCatalog) Env.getCurrentEnv() + .getCatalogMgr().getCatalog(catalogName); + catalog.getCatalogProperty().addProperty("table-default.format-version", "3"); + try { + String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String rowIdSql = "create table " + rowIdTable + " (_row_id bigint)"; + LogicalPlan rowIdPlan = parseStmt(rowIdSql); + Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext)); + Assertions.assertFalse(catalog.getCatalog().tableExists(TableIdentifier.of(dbName, rowIdTable))); + + String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String formatV2Sql = "create table " + formatV2Table + + " (_row_id bigint) properties('format-version'='2')"; + LogicalPlan formatV2Plan = parseStmt(formatV2Sql); + Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand); + Assertions.assertDoesNotThrow( + () -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext)); + } finally { + catalog.getCatalogProperty().deleteProperty("table-default.format-version"); + } + } + + @Test + public void testIcebergV3CtasRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + String ctasTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String ctasSql = "create table " + ctasTable + + " properties('format-version'='3') as select 1 as _row_id"; + LogicalPlan ctasPlan = parseStmt(ctasSql); + Assertions.assertTrue(ctasPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) ctasPlan).validateCreateTableAsSelect( + connectContext, ((CreateTableCommand) ctasPlan).getCtasQuery().get())); + } + @Test public void testIcebergUpdatePlans() throws Exception { useIceberg(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java index 2bf1ef3deb4545..55a3a63a5fabfb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java @@ -17,9 +17,17 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -27,6 +35,7 @@ import org.apache.iceberg.catalog.ViewCatalog; import org.junit.Assert; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.util.Arrays; @@ -114,4 +123,50 @@ public void testListTableNamesFiltersViewsWhenRestViewEnabled() { Assert.assertEquals(Collections.singletonList("DORIS_HORIZON_T"), tableNames); } + + @Test + public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws Exception { + Map catalogProps = new HashMap<>(); + catalogProps.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "3"); + IcebergExternalCatalog dorisCatalog = mockHmsCatalog(catalogProps); + Catalog icebergCatalog = Mockito.mock(Catalog.class, + Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); + IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); + + ExternalDatabase dorisDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(dorisDb.getRemoteName()).thenReturn("db"); + Mockito.when(dorisDb.getTableNullable("tbl")).thenReturn(null); + Mockito.doReturn(dorisDb).when(dorisCatalog).getDbNullable("db"); + Mockito.when(dorisCatalog.getName()).thenReturn("iceberg_catalog"); + Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenReturn(false); + + CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class); + Map tableProps = new HashMap<>(); + Mockito.when(createTableInfo.getDbName()).thenReturn("db"); + Mockito.when(createTableInfo.getTableName()).thenReturn("tbl"); + Mockito.when(createTableInfo.isIfNotExists()).thenReturn(false); + Mockito.when(createTableInfo.getColumns()).thenReturn(Collections.singletonList( + new Column("id", Type.INT, true))); + Mockito.when(createTableInfo.getProperties()).thenReturn(tableProps); + + ops.performCreateTable(createTableInfo); + + Mockito.verify(createTableInfo).validateIcebergRowLineageColumns(3); + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(icebergCatalog).createTable(Mockito.eq(TableIdentifier.of("db", "tbl")), + Mockito.any(Schema.class), Mockito.any(PartitionSpec.class), propsCaptor.capture()); + Assert.assertFalse(propsCaptor.getValue().containsKey(TableProperties.FORMAT_VERSION)); + Assert.assertEquals(3, IcebergUtils.getEffectiveIcebergFormatVersion( + propsCaptor.getValue(), catalogProps)); + } + + private IcebergExternalCatalog mockHmsCatalog(Map catalogProperties) { + IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + }); + Mockito.when(dorisCatalog.getProperties()).thenReturn(catalogProperties); + Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_HMS); + Mockito.when(dorisCatalog.getCatalogProperty()).thenReturn(new CatalogProperty(null, Collections.emptyMap())); + return dorisCatalog; + } } From 64be7f32d10c2e0728aa832e1ca53436f18d9006 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 10:32:16 +0800 Subject: [PATCH 05/24] [improvement](paimon) Support IOManager for JNI reads (#65332) Paimon JNI reads created `TableRead` without `withIOManager`, so Paimon primary-key merge reads could not spill through Paimon IOManager. This PR wires Doris catalog properties through FE and BE into the Java scanner, creates the Paimon IOManager for JNI reads, and closes it on scanner close or open failure. The same IOManager option/default temp-dir handling is implemented in both Paimon JNI scan paths: - legacy `be/src/format/table/paimon_jni_reader.cpp` - Format V2 `be/src/format_v2/jni/paimon_jni_reader.cpp` --- be/src/format/table/paimon_jni_reader.cpp | 26 +++++++++++++++++++++++ be/src/format/table/paimon_jni_reader.h | 2 ++ 2 files changed, 28 insertions(+) diff --git a/be/src/format/table/paimon_jni_reader.cpp b/be/src/format/table/paimon_jni_reader.cpp index c5de5b35eac7e3..063755d2be58b7 100644 --- a/be/src/format/table/paimon_jni_reader.cpp +++ b/be/src/format/table/paimon_jni_reader.cpp @@ -18,10 +18,14 @@ #include "format/table/paimon_jni_reader.h" #include +#include +#include #include "core/types.h" #include "runtime/descriptors.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "util/string_util.h" namespace doris { class RuntimeProfile; class RuntimeState; @@ -31,8 +35,14 @@ class Block; namespace doris { #include "common/compile_check_begin.h" +namespace { +constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; +} // namespace + const std::string PaimonJniReader::PAIMON_OPTION_PREFIX = "paimon."; const std::string PaimonJniReader::HADOOP_OPTION_PREFIX = "hadoop."; +const std::string PaimonJniReader::DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; +const std::string PaimonJniReader::DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; PaimonJniReader::PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, @@ -75,6 +85,22 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d params[PAIMON_OPTION_PREFIX + kv.first] = kv.second; } } + const std::string enable_io_manager_key = + PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; + const std::string io_manager_tmp_dir_key = + PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; + auto enable_io_manager_it = params.find(enable_io_manager_key); + if (enable_io_manager_it != params.end() && iequal(enable_io_manager_it->second, "true") && + params.find(io_manager_tmp_dir_key) == params.end()) { + std::vector tmp_dirs; + for (const auto& store_path : state->exec_env()->store_paths()) { + tmp_dirs.push_back(store_path.path + "/" + + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + } + DORIS_CHECK(!tmp_dirs.empty()); + // Paimon owns its paimon-* children; Doris only supplies stable storage-root parents. + params[io_manager_tmp_dir_key] = join(tmp_dirs, ":"); + } // Prefer hadoop conf from scan node level (range_params->properties) over split level // to avoid redundant configuration in each split if (range_params->__isset.properties && !range_params->properties.empty()) { diff --git a/be/src/format/table/paimon_jni_reader.h b/be/src/format/table/paimon_jni_reader.h index 0026ab4118103d..feab10b2d39ca9 100644 --- a/be/src/format/table/paimon_jni_reader.h +++ b/be/src/format/table/paimon_jni_reader.h @@ -48,6 +48,8 @@ class PaimonJniReader : public JniReader { public: static const std::string PAIMON_OPTION_PREFIX; static const std::string HADOOP_OPTION_PREFIX; + static const std::string DORIS_ENABLE_JNI_IO_MANAGER; + static const std::string DORIS_JNI_IO_MANAGER_TMP_DIR; PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, const TFileRangeDesc& range, const TFileScanRangeParams* range_params); From 7a190fe30be1e9f237b312533e61e648707d6c84 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 22:35:42 +0800 Subject: [PATCH 06/24] [fix](iceberg) Allow optional position delete fields without nulls (#65401) Relax Iceberg position delete Parquet reads so Doris can consume delete files where `file_path` or `pos` are declared optional in Parquet metadata but contain no actual null values. The reader now materializes position delete columns as nullable, validates that each batch has no nulls, then strips the nullable wrapper before building delete ranges. Actual null values in `file_path` or `pos` still fail with a corruption error. This covers the legacy Iceberg Parquet reader, the shared delete-file helper used by Iceberg writes, and the format_v2 position delete collector. Some writers, including DuckLake/DuckDB-produced data seen in the field, can produce Iceberg position delete files whose required Iceberg fields are encoded as optional Parquet fields. Doris previously rejected these files before checking whether the data actually contained nulls. --- .../iceberg_delete_file_reader_helper.cpp | 41 ++-- be/src/format_v2/table/iceberg_reader.cpp | 18 +- ...iceberg_delete_file_reader_helper_test.cpp | 179 ++++++++++++++++++ 3 files changed, 223 insertions(+), 15 deletions(-) diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 521c2c07400c44..0a2e79f1022f20 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -29,8 +29,10 @@ #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_dictionary.h" +#include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "exec/common/endian.h" @@ -90,9 +92,27 @@ Status visit_position_delete_block(const Block& block, size_t read_rows, return Status::InternalError("Position delete block is missing required columns"); } - const auto* pos_column = - assert_cast(block.get_by_position(pos_it->second).column.get()); - const auto* path_column = block.get_by_position(path_it->second).column.get(); + ColumnPtr path_column_ptr = block.get_by_position(path_it->second).column; + ColumnPtr pos_column_ptr = block.get_by_position(pos_it->second).column; + // Iceberg permits optional schemas here, but a concrete delete key must never be null. + if (const auto* nullable_col = check_and_get_column(*path_column_ptr); + nullable_col != nullptr) { + if (nullable_col->has_null(0, read_rows)) { + return Status::Corruption( + "Iceberg position delete column file_path contains null values"); + } + path_column_ptr = remove_nullable(path_column_ptr); + } + if (const auto* nullable_col = check_and_get_column(*pos_column_ptr); + nullable_col != nullptr) { + if (nullable_col->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column pos contains null values"); + } + pos_column_ptr = remove_nullable(pos_column_ptr); + } + + const auto* pos_column = assert_cast(pos_column_ptr.get()); + const auto* path_column = path_column_ptr.get(); if (const auto* string_column = check_and_get_column(path_column); string_column != nullptr) { @@ -255,16 +275,15 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi while (!eof) { Block block; if (dictionary_coded) { - block.insert(ColumnWithTypeAndName(ColumnDictI32::create(), - std::make_shared(), - ICEBERG_FILE_PATH)); + block.insert(ColumnWithTypeAndName( + ColumnNullable::create(ColumnDictI32::create(), ColumnUInt8::create()), + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); } else { - block.insert(ColumnWithTypeAndName(ColumnString::create(), - std::make_shared(), - ICEBERG_FILE_PATH)); + block.insert(ColumnWithTypeAndName( + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); } - block.insert(ColumnWithTypeAndName(ColumnInt64::create(), - std::make_shared(), ICEBERG_ROW_POS)); + block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), + ICEBERG_ROW_POS)); size_t read_rows = 0; RETURN_IF_ERROR(reader.get_next_block(&block, &read_rows, &eof)); RETURN_IF_ERROR(visit_position_delete_block(block, read_rows, visitor)); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index cc2be83f39f8e5..7af551dcddb730 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -203,10 +203,20 @@ Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& blo if (read_rows == 0) { return Status::OK(); } - const auto& file_path_column = assert_cast( - *remove_nullable((block.get_by_position(ICEBERG_FILE_PATH_BLOCK_POSITION).column))); - const auto& pos_column = assert_cast( - *remove_nullable(block.get_by_position(ICEBERG_ROW_POS_BLOCK_POSITION).column)); + const auto& file_path_column_ptr = + block.get_by_position(ICEBERG_FILE_PATH_BLOCK_POSITION).column; + const auto& pos_column_ptr = block.get_by_position(ICEBERG_ROW_POS_BLOCK_POSITION).column; + if (const auto* nullable_column = check_and_get_column(*file_path_column_ptr); + nullable_column != nullptr && nullable_column->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column file_path contains null values"); + } + if (const auto* nullable_column = check_and_get_column(*pos_column_ptr); + nullable_column != nullptr && nullable_column->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column pos contains null values"); + } + const auto& file_path_column = + assert_cast(*remove_nullable(file_path_column_ptr)); + const auto& pos_column = assert_cast(*remove_nullable(pos_column_ptr)); for (size_t row = 0; row < read_rows; ++row) { const auto file_path = file_path_column.get_data_at(row).to_string(); (*_rows_by_data_file)[file_path].push_back(pos_column.get_element(row)); diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp index e149d69328810d..40d4a93639aea8 100644 --- a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp +++ b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp @@ -17,9 +17,19 @@ #include "format/table/iceberg_delete_file_reader_helper.h" +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -49,6 +59,112 @@ class CollectPositionDeleteVisitor final : public IcebergPositionDeleteVisitor { size_t total_rows = 0; }; +TFileScanRangeParams make_local_parquet_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return scan_params; +} + +TIcebergDeleteFileDesc make_iceberg_deletion_vector(const std::string& path, int64_t offset, + int64_t size) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(3); + delete_file.__set_path(path); + delete_file.__set_content_offset(offset); + delete_file.__set_content_size_in_bytes(size); + return delete_file; +} + +int64_t write_iceberg_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + roaring::Roaring64Map rows; + for (const auto position : deleted_positions) { + rows.add(position); + } + + const size_t bitmap_size = rows.getSizeInBytes(); + std::vector blob(4 + 4 + bitmap_size + 4); + rows.write(blob.data() + 8); + + const uint32_t total_length = static_cast(4 + bitmap_size); + BigEndian::Store32(blob.data(), total_length); + constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; + memcpy(blob.data() + 4, DV_MAGIC, 4); + BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + + std::ofstream output(file_path, std::ios::binary); + EXPECT_TRUE(output.is_open()); + output.write(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(output.good()); + return static_cast(blob.size()); +} + +std::string temp_parquet_path(const std::string& filename) { + return (std::filesystem::temp_directory_path() / filename).string(); +} + +void write_position_delete_parquet(const std::string& path, + const std::vector>& file_paths, + const std::vector>& positions) { + ASSERT_EQ(file_paths.size(), positions.size()); + + arrow::StringBuilder path_builder; + arrow::Int64Builder pos_builder; + for (size_t i = 0; i < file_paths.size(); ++i) { + if (file_paths[i].has_value()) { + ASSERT_TRUE(path_builder.Append(*file_paths[i]).ok()); + } else { + ASSERT_TRUE(path_builder.AppendNull().ok()); + } + if (positions[i].has_value()) { + ASSERT_TRUE(pos_builder.Append(*positions[i]).ok()); + } else { + ASSERT_TRUE(pos_builder.AppendNull().ok()); + } + } + + std::shared_ptr path_array; + std::shared_ptr pos_array; + ASSERT_TRUE(path_builder.Finish(&path_array).ok()); + ASSERT_TRUE(pos_builder.Finish(&pos_array).ok()); + + auto schema = arrow::schema({arrow::field("file_path", arrow::utf8(), true), + arrow::field("pos", arrow::int64(), true)}); + auto table = arrow::Table::Make(schema, {path_array, pos_array}); + + std::shared_ptr output; + ASSERT_TRUE(arrow::io::FileOutputStream::Open(path).Value(&output).ok()); + PARQUET_THROW_NOT_OK( + parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1024)); +} + +IcebergDeleteFileReaderOptions make_delete_file_reader_options( + RuntimeState* state, RuntimeProfile* profile, const TFileScanRangeParams* scan_params, + io::IOContext* io_ctx) { + return { + .state = state, + .profile = profile, + .scan_params = scan_params, + .io_ctx = io_ctx, + }; +} + +IcebergDeleteFileReaderOptions delete_reader_options(RuntimeState* runtime_state, + RuntimeProfile* profile, + TFileScanRangeParams* scan_params, + IcebergDeleteFileIOContext* io_context, + FileMetaCache* meta_cache) { + IcebergDeleteFileReaderOptions options; + options.state = runtime_state; + options.profile = profile; + options.scan_params = scan_params; + options.io_ctx = &io_context->io_ctx; + options.meta_cache = meta_cache; + options.batch_size = 1024; + return options; +} + } // namespace TEST(IcebergDeleteFileReaderHelperTest, BuildDeleteFileRange) { @@ -107,4 +223,67 @@ TEST(IcebergDeleteFileReaderHelperTest, ReadMixedEncodingParquetPositionDeleteFi EXPECT_EQ(it->second, expected_positions); } +TEST(IcebergDeleteFileReaderHelperTest, ReadOptionalParquetPositionDeleteColumnsWithoutNulls) { + const auto delete_file_path = + temp_parquet_path("iceberg_optional_position_delete_without_nulls.parquet"); + write_position_delete_parquet(delete_file_path, + {std::string(kTargetDataFilePath), + std::string(kTargetDataFilePath), "s3://other/file.parquet"}, + {3, 9, 11}); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); + FileMetaCache meta_cache(1024); + IcebergDeleteFileIOContext io_context(&runtime_state); + + TFileScanRangeParams scan_params; + scan_params.file_type = TFileType::FILE_LOCAL; + scan_params.format_type = TFileFormatType::FORMAT_PARQUET; + + TIcebergDeleteFileDesc delete_file; + delete_file.path = delete_file_path; + delete_file.file_format = TFileFormatType::FORMAT_PARQUET; + delete_file.__isset.file_format = true; + + CollectPositionDeleteVisitor visitor; + auto options = + delete_reader_options(&runtime_state, &profile, &scan_params, &io_context, &meta_cache); + auto st = read_iceberg_position_delete_file(delete_file, options, &visitor); + + std::filesystem::remove(delete_file_path); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(visitor.total_rows, 3); + EXPECT_EQ(visitor.delete_rows[kTargetDataFilePath], std::vector({3, 9})); +} + +TEST(IcebergDeleteFileReaderHelperTest, RejectParquetPositionDeleteColumnsWithActualNulls) { + const auto delete_file_path = temp_parquet_path("iceberg_position_delete_with_nulls.parquet"); + write_position_delete_parquet(delete_file_path, + {std::string(kTargetDataFilePath), std::nullopt}, {3, 9}); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); + FileMetaCache meta_cache(1024); + IcebergDeleteFileIOContext io_context(&runtime_state); + + TFileScanRangeParams scan_params; + scan_params.file_type = TFileType::FILE_LOCAL; + scan_params.format_type = TFileFormatType::FORMAT_PARQUET; + + TIcebergDeleteFileDesc delete_file; + delete_file.path = delete_file_path; + delete_file.file_format = TFileFormatType::FORMAT_PARQUET; + delete_file.__isset.file_format = true; + + CollectPositionDeleteVisitor visitor; + auto options = + delete_reader_options(&runtime_state, &profile, &scan_params, &io_context, &meta_cache); + auto st = read_iceberg_position_delete_file(delete_file, options, &visitor); + + std::filesystem::remove(delete_file_path); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("file_path contains null values"), std::string::npos) + << st.to_string(); +} + } // namespace doris From e0c83efe29dd0b9a3cef2f781e3f994873bfcaa1 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 14 Jul 2026 22:40:53 +0800 Subject: [PATCH 07/24] [feature](iceberg) Support Iceberg position deletes system table (#65135) Checkpoint conclusions: Goal and scope: the patch adds native Iceberg $position_deletes metadata-table support across FE system-table resolution, split construction, Thrift scan metadata, BE scanner-v1/scanner-v2 readers, regression coverage, and unit tests. I did not find an out-of-scope behavioral change needing a new comment. Existing review context: prior comments already covered the earlier V1 IOContext/profile issue, smooth-upgrade guard, Avro unsupported format, partition JSON/type/evolution/rename concerns, and the ORC row assertion typo. I did not duplicate those threads. FE/BE protocol and compatibility: native $position_deletes ranges use top-level Iceberg content to route into the new readers, while ordinary Iceberg data ranges keep delete-file descriptors nested and do not match that route. The smooth-upgrade backend guard now runs before native position-delete ranges are generated. Data correctness and nullability: partition values are mapped by Iceberg partition field ID into the metadata-table partition struct, missing evolved fields become null, unsupported non-null binary/fixed/UUID partition values fail instead of silently producing wrong data, optional absent row payloads materialize as null, and deletion-vector rows are expanded from the referenced data file plus decoded positions. Scanner and lifecycle paths: scanner V1/V2 routing, Parquet/ORC position-delete files, Puffin deletion vectors, EOF/final-batch handling, IOContext propagation, profile/stat accounting, close/reset state, and row-count behavior were reviewed without a substantiated new defect. Tests and coverage: the changed regression and unit tests cover the main feature surface, including scanner V1/V2, Parquet/ORC deletes, V3 deletion-vector metadata, partition evolution/rename, typed partitions, time travel, empty-delete tables, and privilege checks. A possible Puffin-DV split concern was dismissed because Doris depends on Iceberg 1.10.1, where FileFormat.PUFFIN is non-splittable and BaseContentScanTask.split(...) returns the original task for non-splittable files. Concurrency, persistence, transactions, and config: no new shared mutable concurrency path, edit-log/persistence path, transaction/write path, or user-facing config was introduced by this read-only metadata scan feature. Observability/performance: existing scanner/profile counters remain in use; I did not find a concrete additional metrics requirement or hot-path regression. --- be/src/exec/scan/file_scanner.cpp | 38 +- be/src/exec/scan/file_scanner_v2.cpp | 17 +- ...eberg_position_delete_sys_table_reader.cpp | 614 ++++++++++++++++ ...iceberg_position_delete_sys_table_reader.h | 111 +++ ...eberg_position_delete_sys_table_reader.cpp | 596 ++++++++++++++++ ...iceberg_position_delete_sys_table_reader.h | 86 +++ be/test/exec/scan/file_scanner_v2_test.cpp | 32 + ..._position_delete_sys_table_reader_test.cpp | 608 ++++++++++++++++ .../iceberg/IcebergExternalTable.java | 13 - .../iceberg/source/IcebergScanNode.java | 296 +++++++- .../iceberg/source/IcebergSplit.java | 17 + .../datasource/systable/IcebergSysTable.java | 15 +- .../iceberg/source/IcebergScanNodeTest.java | 136 ++++ .../systable/IcebergSysTableResolverTest.java | 15 +- gensrc/thrift/PlanNodes.thrift | 4 + ...est_iceberg_position_deletes_sys_table.out | 4 + ..._iceberg_position_deletes_sys_table.groovy | 659 ++++++++++++++++++ .../iceberg/test_iceberg_sys_table.groovy | 27 +- 18 files changed, 3240 insertions(+), 48 deletions(-) create mode 100644 be/src/format/table/iceberg_position_delete_sys_table_reader.cpp create mode 100644 be/src/format/table/iceberg_position_delete_sys_table_reader.h create mode 100644 be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp create mode 100644 be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h create mode 100644 be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 188c1f6efdb6ef..5c7beb3e87e268 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -68,6 +68,7 @@ #include "format/table/hive_reader.h" #include "format/table/hudi_jni_reader.h" #include "format/table/hudi_reader.h" +#include "format/table/iceberg_position_delete_sys_table_reader.h" #include "format/table/iceberg_reader.h" #include "format/table/iceberg_sys_table_jni_reader.h" #include "format/table/lakesoul_jni_reader.h" @@ -99,6 +100,20 @@ namespace doris { #include "common/compile_check_begin.h" using namespace ErrorCode; +namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + +bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { + return range.__isset.table_format_params && + range.table_format_params.table_format_type == "iceberg" && + range.table_format_params.__isset.iceberg_params && + range.table_format_params.iceberg_params.__isset.content && + (range.table_format_params.iceberg_params.content == kIcebergPositionDeleteContent || + range.table_format_params.iceberg_params.content == kIcebergDeletionVectorContent); +} +} // namespace + const std::string FileScanner::FileReadBytesProfile = "FileReadBytes"; const std::string FileScanner::FileReadTimeProfile = "FileReadTime"; @@ -1060,6 +1075,7 @@ Status FileScanner::_get_next_reader() { // create reader for specific format Status init_status = Status::OK(); TFileFormatType::type format_type = _get_current_format_type(); + const bool is_position_deletes_sys_table = is_iceberg_position_deletes_sys_table(range); // for compatibility, this logic is deprecated in 3.1 if (format_type == TFileFormatType::FORMAT_JNI && range.__isset.table_format_params) { if (range.table_format_params.table_format_type == "paimon" && @@ -1149,6 +1165,15 @@ Status FileScanner::_get_next_reader() { auto file_meta_cache_ptr = _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache() : nullptr; + if (is_position_deletes_sys_table) { + auto reader = IcebergPositionDeleteSysTableReader::create_unique( + _file_slot_descs, _state, _profile, range, _params, _io_ctx, + file_meta_cache_ptr); + init_status = reader->init_reader(); + _cur_reader = std::move(reader); + need_to_get_parsed_schema = false; + break; + } std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, _state->query_options().batch_size, const_cast(&_state->timezone_obj()), _io_ctx, _state, @@ -1174,6 +1199,15 @@ Status FileScanner::_get_next_reader() { auto file_meta_cache_ptr = _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache() : nullptr; + if (is_position_deletes_sys_table) { + auto reader = IcebergPositionDeleteSysTableReader::create_unique( + _file_slot_descs, _state, _profile, range, _params, _io_ctx, + file_meta_cache_ptr); + init_status = reader->init_reader(); + _cur_reader = std::move(reader); + need_to_get_parsed_schema = false; + break; + } std::unique_ptr orc_reader = OrcReader::create_unique( _profile, _state, *_params, range, _state->query_options().batch_size, _state->timezone(), _io_ctx, file_meta_cache_ptr, @@ -1680,8 +1714,8 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, case TFileFormatType::FORMAT_PARQUET: { std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, 1, - const_cast(&_state->timezone_obj()), _io_ctx, - _state, file_meta_cache_ptr, false); + const_cast(&_state->timezone_obj()), _io_ctx, _state, + file_meta_cache_ptr, false); RETURN_IF_ERROR(parquet_reader->read_by_rows(row_ids)); RETURN_IF_ERROR( diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 91320e080439b9..5b593ccd178604 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -54,6 +54,7 @@ #include "format_v2/jni/trino_connector_jni_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "format_v2/table/iceberg_reader.h" #include "format_v2/table/paimon_reader.h" #include "format_v2/table/remote_doris_reader.h" @@ -70,6 +71,9 @@ namespace doris { namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + std::string table_format_name(const TFileRangeDesc& range) { return range.__isset.table_format_params ? range.table_format_params.table_format_type : "NotSet"; @@ -110,6 +114,15 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { table_format == "max_compute" || table_format == "trino_connector"; } +bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { + return range.__isset.table_format_params && + range.table_format_params.table_format_type == "iceberg" && + range.table_format_params.__isset.iceberg_params && + range.table_format_params.iceberg_params.__isset.content && + (range.table_format_params.iceberg_params.content == kIcebergPositionDeleteContent || + range.table_format_params.iceberg_params.content == kIcebergDeletionVectorContent); +} + bool is_csv_format(TFileFormatType::type format_type) { switch (format_type) { case TFileFormatType::FORMAT_CSV_PLAIN: @@ -463,7 +476,9 @@ Status FileScannerV2::_create_table_reader_for_format( } else if (table_format == "hive") { *reader = format::hive::HiveReader::create_unique(); } else if (table_format == "iceberg") { - if (get_range_format_type(*_params, range) == TFileFormatType::FORMAT_JNI) { + if (is_iceberg_position_deletes_sys_table(range)) { + *reader = std::make_unique(); + } else if (get_range_format_type(*_params, range) == TFileFormatType::FORMAT_JNI) { *reader = std::make_unique(); } else { *reader = std::make_unique(); diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp new file mode 100644 index 00000000000000..c01027179dd2fd --- /dev/null +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -0,0 +1,614 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/iceberg_position_delete_sys_table_reader.h" + +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/types.h" +#include "format/orc/vorc_reader.h" +#include "format/parquet/schema_desc.h" +#include "format/parquet/vparquet_reader.h" +#include "format/table/parquet_utils.h" +#include "format/table/table_format_reader.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { + +constexpr const char* kFilePathColumn = "file_path"; +constexpr const char* kPosColumn = "pos"; +constexpr const char* kRowColumn = "row"; +constexpr const char* kPartitionColumn = "partition"; +constexpr const char* kSpecIdColumn = "spec_id"; +constexpr const char* kDeleteFilePathColumn = "delete_file_path"; +constexpr const char* kContentOffsetColumn = "content_offset"; +constexpr const char* kContentSizeInBytesColumn = "content_size_in_bytes"; +constexpr const char* kIcebergOrcAttribute = "iceberg.id"; +constexpr int kPositionDeleteContent = 1; + +bool block_has_row(const Block& block, size_t row) { + return block.columns() > 0 && row < block.rows(); +} + +void insert_int64_nullable(MutableColumnPtr& column, const int64_t* value) { + if (value == nullptr) { + parquet_utils::insert_null(column); + } else { + parquet_utils::insert_int64(column, *value); + } +} + +// Fail loudly if the filled output block is malformed. Each output column must be produced by a +// file slot; a projected system-table column that is not backed by a file slot would be skipped +// during fill and left shorter than the rest, producing a block with inconsistent column lengths. +void check_output_columns_aligned(const MutableColumns& columns) { + if (columns.empty()) { + return; + } + const size_t expected_rows = columns.front()->size(); + for (const auto& column : columns) { + DORIS_CHECK(column->size() == expected_rows) + << "Iceberg position delete system table output block has inconsistent column " + "sizes; a projected column is not backed by a file slot"; + } +} + +const ColumnString* get_string_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return check_and_get_column(block.get_by_position(pos).column.get()); +} + +const ColumnInt64* get_int64_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return check_and_get_column(block.get_by_position(pos).column.get()); +} + +const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { + if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { + return nullptr; + } + return field_ptr.field_ptr.get(); +} + +const schema::external::TField* find_current_schema_field(const TFileScanRangeParams* params, + const std::string& name) { + if (params == nullptr || !params->__isset.history_schema_info || + params->history_schema_info.empty()) { + return nullptr; + } + const schema::external::TSchema* schema = ¶ms->history_schema_info.front(); + if (params->__isset.current_schema_id) { + for (const auto& candidate : params->history_schema_info) { + if (candidate.__isset.schema_id && candidate.schema_id == params->current_schema_id) { + schema = &candidate; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return nullptr; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.name && field->name == name) { + return field; + } + } + return nullptr; +} + +template +std::shared_ptr create_position_delete_root_node( + const ReadColumns& read_columns) { + auto root_node = std::make_shared(); + for (const auto& column : read_columns) { + if (column.name == kRowColumn) { + continue; + } + root_node->add_children(column.name, column.name, + TableSchemaChangeHelper::ConstNode::get_instance()); + } + return root_node; +} + +} // namespace + +IcebergPositionDeleteSysTableReader::IcebergPositionDeleteSysTableReader( + const std::vector& file_slot_descs, RuntimeState* state, + RuntimeProfile* profile, const TFileRangeDesc& range, + const TFileScanRangeParams* range_params, std::shared_ptr io_ctx, + FileMetaCache* meta_cache) + : _file_slot_descs(file_slot_descs), + _state(state), + _profile(profile), + _range(range), + _range_params(range_params), + _io_ctx(std::move(io_ctx)) { + _meta_cache = meta_cache; +} + +IcebergPositionDeleteSysTableReader::~IcebergPositionDeleteSysTableReader() = default; + +Status IcebergPositionDeleteSysTableReader::init_reader() { + if (_state == nullptr || _profile == nullptr || _range_params == nullptr || + _io_ctx == nullptr) { + return Status::InvalidArgument( + "invalid Iceberg position delete system table reader context"); + } + if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("Iceberg position delete system table range misses params"); + } + + _iceberg_file_desc = &_range.table_format_params.iceberg_params; + if (!_iceberg_file_desc->__isset.delete_files || _iceberg_file_desc->delete_files.size() != 1) { + return Status::InternalError( + "Iceberg position delete system table range should contain exactly one delete " + "file"); + } + _delete_file_desc = _iceberg_file_desc->delete_files.data(); + if (is_iceberg_deletion_vector(*_delete_file_desc)) { + _delete_file_kind = DeleteFileKind::DELETION_VECTOR; + } else if (_delete_file_desc->__isset.content && + _delete_file_desc->content == kPositionDeleteContent) { + _delete_file_kind = DeleteFileKind::POSITION_DELETE; + } else if (!_delete_file_desc->__isset.content) { + return Status::InternalError( + "Iceberg position delete system table delete file misses content"); + } else { + return Status::InternalError( + "Iceberg position delete system table does not support delete file content {}", + _delete_file_desc->content); + } + _batch_size = _state->batch_size(); + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _init_deletion_vector_reader(); + } + return _init_position_delete_reader(); +} + +Status IcebergPositionDeleteSysTableReader::get_columns( + std::unordered_map* name_to_type, + std::unordered_set* missing_cols) { + missing_cols->clear(); + for (const auto* slot : _file_slot_descs) { + name_to_type->emplace(slot->col_name(), slot->get_data_type_ptr()); + } + return Status::OK(); +} + +bool IcebergPositionDeleteSysTableReader::count_read_rows() { + return _delete_file_kind == DeleteFileKind::POSITION_DELETE; +} + +void IcebergPositionDeleteSysTableReader::_collect_profile_before_close() { + if (_position_reader != nullptr) { + _position_reader->collect_profile_before_close(); + } +} + +Status IcebergPositionDeleteSysTableReader::close() { + if (_position_reader != nullptr) { + RETURN_IF_ERROR(_position_reader->close()); + } + _partition_value.reset(); + _next_dv_position.reset(); + _dv_positions = roaring::Roaring64Map(); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { + if (!_delete_file_desc->__isset.file_format) { + return Status::InternalError("Iceberg position delete file misses file format"); + } + + // `row` is optional in position delete files and expensive to read, so only read it when the + // query actually projects it. Whether the delete file physically stores `row` is decided from + // the reader's own footer/type below, reusing the same reader instance that is initialized + // afterwards so the delete file is opened and its footer parsed only once. + const bool row_requested = _output_column_requested(kRowColumn); + + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_PARQUET) { + auto parquet_reader = + ParquetReader::create_unique(_profile, *_range_params, _range, _batch_size, + const_cast(&_state->timezone_obj()), + _io_ctx, _state, _meta_cache); + + const FieldDescriptor* schema = nullptr; + int row_index = -1; + if (row_requested) { + RETURN_IF_ERROR(parquet_reader->get_file_metadata_schema(&schema)); + DORIS_CHECK(schema != nullptr); + row_index = schema->get_column_index(kRowColumn); + } + const bool read_row = row_requested && row_index >= 0; + _init_read_columns(read_row); + std::vector read_column_names; + read_column_names.reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + read_column_names.push_back(column.name); + } + + std::shared_ptr table_info_node = + TableSchemaChangeHelper::ConstNode::get_instance(); + if (read_row) { + const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); + if (table_row_field == nullptr) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + const auto* file_row_field = schema->get_column(static_cast(row_index)); + std::shared_ptr row_node; + // The branch-4.1 helper selects ID or name-mapping mode up front instead of doing so + // inside each nested node, so seed that mode from the projected row field. + const bool exist_field_id = file_row_field->field_id != -1; + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( + *table_row_field, *file_row_field, exist_field_id, row_node)); + auto root_node = create_position_delete_root_node(_read_columns); + root_node->add_children(kRowColumn, file_row_field->name, row_node); + table_info_node = std::move(root_node); + } + // branch-4.1's legacy reader API predates ReaderInitContext. Position-delete scans have no + // predicates, so pass empty filter state while preserving the schema mapping built above. + VExprContextSPtrs conjuncts; + phmap::flat_hash_map>> + slot_id_to_predicates; + RETURN_IF_ERROR(parquet_reader->init_reader( + read_column_names, &_read_col_name_to_block_idx, conjuncts, slot_id_to_predicates, + nullptr, nullptr, nullptr, nullptr, nullptr, std::move(table_info_node), false)); + _position_reader = std::move(parquet_reader); + return Status::OK(); + } + + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_ORC) { + auto orc_reader = + OrcReader::create_unique(_profile, _state, *_range_params, _range, _batch_size, + _state->timezone(), _io_ctx, _meta_cache); + + const orc::Type* row_type = nullptr; + if (row_requested) { + const orc::Type* root_type = nullptr; + RETURN_IF_ERROR(orc_reader->get_file_type(&root_type)); + DORIS_CHECK(root_type != nullptr); + for (uint64_t i = 0; i < root_type->getSubtypeCount(); ++i) { + if (root_type->getFieldName(i) == kRowColumn) { + row_type = root_type->getSubtype(i); + break; + } + } + } + const bool read_row = row_requested && row_type != nullptr; + _init_read_columns(read_row); + std::vector read_column_names; + read_column_names.reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + read_column_names.push_back(column.name); + } + + std::shared_ptr table_info_node = + TableSchemaChangeHelper::ConstNode::get_instance(); + if (read_row) { + const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); + if (table_row_field == nullptr) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + std::shared_ptr row_node; + const bool exist_field_id = row_type->hasAttributeKey(kIcebergOrcAttribute); + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id( + *table_row_field, row_type, kIcebergOrcAttribute, exist_field_id, row_node)); + auto root_node = create_position_delete_root_node(_read_columns); + root_node->add_children(kRowColumn, kRowColumn, row_node); + table_info_node = std::move(root_node); + } + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(orc_reader->init_reader(&read_column_names, &_read_col_name_to_block_idx, + conjuncts, false, nullptr, nullptr, nullptr, + nullptr, std::move(table_info_node))); + _position_reader = std::move(orc_reader); + return Status::OK(); + } + + return Status::NotSupported("Unsupported Iceberg position delete file format {}", + _delete_file_desc->file_format); +} + +Status IcebergPositionDeleteSysTableReader::_init_deletion_vector_reader() { + if (!_delete_file_desc->__isset.referenced_data_file_path || + _delete_file_desc->referenced_data_file_path.empty()) { + return Status::InternalError("Iceberg deletion vector misses referenced data file path"); + } + + IcebergDeleteFileReaderOptions options; + options.state = _state; + options.profile = _profile; + options.scan_params = _range_params; + options.io_ctx = _io_ctx.get(); + options.meta_cache = _meta_cache; + if (_range.__isset.fs_name) { + options.fs_name = &_range.fs_name; + } + + _dv_positions = roaring::Roaring64Map(); + RETURN_IF_ERROR(read_iceberg_deletion_vector(*_delete_file_desc, options, &_dv_positions)); + _next_dv_position.emplace(_dv_positions.begin()); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::get_next_block(Block* block, size_t* read_rows, + bool* eof) { + DORIS_CHECK(_io_ctx != nullptr); + if (_io_ctx->should_stop) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _append_deletion_vector_block(block, read_rows, eof); + } + + if (_position_reader == nullptr) { + return Status::InternalError("Iceberg position delete reader is not initialized"); + } + + while (true) { + Block delete_block = _create_delete_block(); + size_t delete_rows = 0; + bool position_reader_eof = false; + RETURN_IF_ERROR(_position_reader->get_next_block(&delete_block, &delete_rows, + &position_reader_eof)); + if (delete_rows > 0) { + RETURN_IF_ERROR( + _append_position_delete_block(block, delete_block, delete_rows, read_rows)); + *eof = position_reader_eof; + return Status::OK(); + } + if (position_reader_eof) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + } +} + +Status IcebergPositionDeleteSysTableReader::_append_position_delete_block(Block* output_block, + const Block& delete_block, + size_t delete_rows, + size_t* appended_rows) { + auto columns_guard = output_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = output_block->get_name_to_pos_map(); + + for (size_t row = 0; row < delete_rows; ++row) { + for (const auto* slot : _file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, &delete_block, row, 0)); + } + } + // Every output column must be produced by a file slot above; a projected system-table column + // that is not backed by a file slot would be left short and yield a malformed block with + // inconsistent column lengths. + check_output_columns_aligned(columns); + *appended_rows = delete_rows; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_append_deletion_vector_block(Block* block, + size_t* read_rows, + bool* eof) { + if (!_next_dv_position.has_value() || *_next_dv_position == _dv_positions.end()) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + const size_t rows_limit = std::max(_batch_size, 1); + + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = block->get_name_to_pos_map(); + + size_t rows = 0; + while (rows < rows_limit && *_next_dv_position != _dv_positions.end()) { + const uint64_t dv_pos = **_next_dv_position; + for (const auto* slot : _file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, nullptr, 0, dv_pos)); + } + ++(*_next_dv_position); + ++rows; + } + check_output_columns_aligned(columns); + *read_rows = rows; + *eof = *_next_dv_position == _dv_positions.end(); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_append_sys_column(MutableColumnPtr& column, + const SlotDescriptor& slot, + const Block* delete_block, + size_t source_row, uint64_t dv_pos) { + const std::string& name = slot.col_name(); + if (name == kFilePathColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_string(column, _delete_file_desc->referenced_data_file_path); + return Status::OK(); + } + const auto* path_column = get_string_column(*delete_block, kFilePathColumn); + if (path_column == nullptr || !block_has_row(*delete_block, source_row)) { + return Status::InternalError("Iceberg position delete file_path column is missing"); + } + parquet_utils::insert_string(column, path_column->get_data_at(source_row).to_string()); + return Status::OK(); + } + + if (name == kPosColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_int64(column, cast_set(dv_pos)); + return Status::OK(); + } + const auto* pos_column = get_int64_column(*delete_block, kPosColumn); + if (pos_column == nullptr || !block_has_row(*delete_block, source_row)) { + return Status::InternalError("Iceberg position delete pos column is missing"); + } + parquet_utils::insert_int64(column, pos_column->get_element(source_row)); + return Status::OK(); + } + + if (name == kRowColumn) { + if (delete_block != nullptr) { + auto row_pos = delete_block->get_position_by_name(kRowColumn); + if (row_pos >= 0) { + const auto& row_column = *delete_block->get_by_position(row_pos).column; + if (source_row < row_column.size()) { + column->insert_from(row_column, source_row); + return Status::OK(); + } + } + } + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (name == kPartitionColumn) { + return _append_partition_column(column, slot); + } + + if (name == kSpecIdColumn) { + if (_iceberg_file_desc->__isset.partition_spec_id) { + parquet_utils::insert_int32(column, + cast_set(_iceberg_file_desc->partition_spec_id)); + } else { + parquet_utils::insert_null(column); + } + return Status::OK(); + } + + if (name == kDeleteFilePathColumn) { + parquet_utils::insert_string(column, _delete_file_output_path()); + return Status::OK(); + } + + if (name == kContentOffsetColumn) { + const int64_t* value = _delete_file_desc->__isset.content_offset + ? &_delete_file_desc->content_offset + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + if (name == kContentSizeInBytesColumn) { + const int64_t* value = _delete_file_desc->__isset.content_size_in_bytes + ? &_delete_file_desc->content_size_in_bytes + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + return Status::InternalError("Unknown Iceberg position delete system table column: {}", name); +} + +Status IcebergPositionDeleteSysTableReader::_append_partition_column(MutableColumnPtr& column, + const SlotDescriptor& slot) { + if (!_iceberg_file_desc->__isset.partition_data_json || + _iceberg_file_desc->partition_data_json.empty()) { + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (!_partition_value) { + auto partition_value = slot.get_data_type_ptr()->create_column(); + auto serde = slot.get_data_type_ptr()->get_serde(); + StringRef partition_data(_iceberg_file_desc->partition_data_json.data(), + _iceberg_file_desc->partition_data_json.size()); + DataTypeSerDe::FormatOptions options = DataTypeSerDe::get_default_format_options(); + RETURN_IF_ERROR(serde->from_string(partition_data, *partition_value, options)); + DORIS_CHECK(partition_value->size() == 1); + _partition_value = std::move(partition_value); + } + column->insert_from(*_partition_value, 0); + return Status::OK(); +} + +Block IcebergPositionDeleteSysTableReader::_create_delete_block() const { + Block block; + for (const auto& column : _read_columns) { + block.insert(ColumnWithTypeAndName(column.type->create_column(), column.type, column.name)); + } + return block; +} + +bool IcebergPositionDeleteSysTableReader::_output_column_requested(const std::string& name) const { + return std::any_of(_file_slot_descs.begin(), _file_slot_descs.end(), + [&name](const SlotDescriptor* slot) { return slot->col_name() == name; }); +} + +void IcebergPositionDeleteSysTableReader::_init_read_columns(bool read_row) { + _read_columns.clear(); + _read_columns.push_back({kFilePathColumn, std::make_shared()}); + _read_columns.push_back({kPosColumn, std::make_shared()}); + if (read_row) { + for (const auto* slot : _file_slot_descs) { + if (slot->col_name() == kRowColumn) { + _read_columns.push_back({kRowColumn, slot->get_data_type_ptr()}); + break; + } + } + } + + _read_col_name_to_block_idx.clear(); + for (size_t i = 0; i < _read_columns.size(); ++i) { + _read_col_name_to_block_idx.emplace(_read_columns[i].name, cast_set(i)); + } +} + +const std::string& IcebergPositionDeleteSysTableReader::_delete_file_output_path() const { + if (_delete_file_desc->__isset.original_path && !_delete_file_desc->original_path.empty()) { + return _delete_file_desc->original_path; + } + return _delete_file_desc->path; +} + +} // namespace doris diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.h b/be/src/format/table/iceberg_position_delete_sys_table_reader.h new file mode 100644 index 00000000000000..ae16486395f97a --- /dev/null +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/factory_creator.h" +#include "common/status.h" +#include "core/block/column_with_type_and_name.h" +#include "format/generic_reader.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "runtime/descriptors.h" + +namespace doris { + +class FileMetaCache; +class RuntimeProfile; +class RuntimeState; + +class IcebergPositionDeleteSysTableReader : public GenericReader { + ENABLE_FACTORY_CREATOR(IcebergPositionDeleteSysTableReader); + +public: + IcebergPositionDeleteSysTableReader(const std::vector& file_slot_descs, + RuntimeState* state, RuntimeProfile* profile, + const TFileRangeDesc& range, + const TFileScanRangeParams* range_params, + std::shared_ptr io_ctx, + FileMetaCache* meta_cache); + ~IcebergPositionDeleteSysTableReader() override; + + Status init_reader(); + Status get_next_block(Block* block, size_t* read_rows, bool* eof) override; + Status get_columns(std::unordered_map* name_to_type, + std::unordered_set* missing_cols) override; + void set_batch_size(size_t batch_size) override { _batch_size = batch_size; } + size_t get_batch_size() const override { return _batch_size; } + bool count_read_rows() override; + Status close() override; + +protected: + void _collect_profile_before_close() override; + +private: + enum class DeleteFileKind { + POSITION_DELETE, + DELETION_VECTOR, + }; + + struct ReadColumn { + std::string name; + DataTypePtr type; + }; + + Status _init_position_delete_reader(); + Status _init_deletion_vector_reader(); + Status _append_position_delete_block(Block* output_block, const Block& delete_block, + size_t delete_rows, size_t* appended_rows); + Status _append_deletion_vector_block(Block* block, size_t* read_rows, bool* eof); + Status _append_sys_column(MutableColumnPtr& column, const SlotDescriptor& slot, + const Block* delete_block, size_t source_row, uint64_t dv_pos); + Status _append_partition_column(MutableColumnPtr& column, const SlotDescriptor& slot); + Block _create_delete_block() const; + bool _output_column_requested(const std::string& name) const; + void _init_read_columns(bool read_row); + const std::string& _delete_file_output_path() const; + + const std::vector& _file_slot_descs; + RuntimeState* _state = nullptr; + RuntimeProfile* _profile = nullptr; + const TFileRangeDesc& _range; + const TFileScanRangeParams* _range_params = nullptr; + + std::shared_ptr _io_ctx; + const TIcebergFileDesc* _iceberg_file_desc = nullptr; + const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; + DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; + + size_t _batch_size = 102400; + std::unique_ptr _position_reader; + std::vector _read_columns; + std::unordered_map _read_col_name_to_block_idx; + ColumnPtr _partition_value; + roaring::Roaring64Map _dv_positions; + std::optional _next_dv_position; +}; + +} // namespace doris diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp new file mode 100644 index 00000000000000..ef13f585fb6a6d --- /dev/null +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -0,0 +1,596 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/block/block.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/types.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/parquet_utils.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { + +namespace { + +constexpr const char* kFilePathColumn = "file_path"; +constexpr const char* kPosColumn = "pos"; +constexpr const char* kRowColumn = "row"; +constexpr const char* kPartitionColumn = "partition"; +constexpr const char* kSpecIdColumn = "spec_id"; +constexpr const char* kDeleteFilePathColumn = "delete_file_path"; +constexpr const char* kContentOffsetColumn = "content_offset"; +constexpr const char* kContentSizeInBytesColumn = "content_size_in_bytes"; +constexpr int kPositionDeleteContent = 1; +constexpr int32_t kDeleteFilePathFieldId = 2147483546; +constexpr int32_t kDeleteFilePosFieldId = 2147483545; +constexpr int32_t kDeleteFileRowFieldId = 2147483544; + +bool block_has_row(const Block& block, size_t row) { + return block.columns() > 0 && row < block.rows(); +} + +void insert_int64_nullable(MutableColumnPtr& column, const int64_t* value) { + if (value == nullptr) { + parquet_utils::insert_null(column); + } else { + parquet_utils::insert_int64(column, *value); + } +} + +// Fail loudly if the filled output block is malformed. Each output column must be produced by a +// file slot; a projected system-table column that is not backed by a file slot would be skipped +// during fill and left shorter than the rest, producing a block with inconsistent column lengths. +void check_output_columns_aligned(const MutableColumns& columns) { + if (columns.empty()) { + return; + } + const size_t expected_rows = columns.front()->size(); + for (const auto& column : columns) { + DORIS_CHECK(column->size() == expected_rows) + << "Iceberg position delete system table output block has inconsistent column " + "sizes; a projected column is not backed by a file slot"; + } +} + +const IColumn* get_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return block.get_by_position(pos).column.get(); +} + +const IColumn* nested_column(const IColumn* column) { + if (column == nullptr) { + return nullptr; + } + if (const auto* nullable = check_and_get_column(column)) { + return nullable->get_nested_column_ptr().get(); + } + return column; +} + +bool column_is_null_at(const IColumn* column, size_t row) { + const auto* nullable = check_and_get_column(column); + return nullable != nullptr && nullable->is_null_at(row); +} + +const ColumnString* get_string_column(const IColumn* column) { + return check_and_get_column(nested_column(column)); +} + +const ColumnInt64* get_int64_column(const IColumn* column) { + return check_and_get_column(nested_column(column)); +} + +ColumnDefinition build_delete_file_column(const std::string& name, DataTypePtr type) { + ColumnDefinition column; + column.identifier = Field::create_field(name); + column.name = name; + column.type = std::move(type); + return column; +} + +void set_iceberg_delete_field_id(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + if (column->name == kFilePathColumn) { + column->identifier = Field::create_field(kDeleteFilePathFieldId); + } else if (column->name == kPosColumn) { + column->identifier = Field::create_field(kDeleteFilePosFieldId); + } else if (column->name == kRowColumn && !column->has_identifier_field_id()) { + column->identifier = Field::create_field(kDeleteFileRowFieldId); + } +} + +bool has_field_id(const std::vector& schema) { + for (const auto& field : schema) { + if (!field.has_identifier_field_id()) { + return false; + } + if (!has_field_id(field.children)) { + return false; + } + } + return true; +} + +class PositionDeleteFileTableReader final : public format::TableReader { +protected: + format::TableColumnMappingMode mapping_mode() const override { + return !_data_reader.file_schema.empty() && has_field_id(_data_reader.file_schema) + ? format::TableColumnMappingMode::BY_FIELD_ID + : format::TableColumnMappingMode::BY_NAME; + } +}; + +} // namespace + +IcebergPositionDeleteSysTableV2Reader::~IcebergPositionDeleteSysTableV2Reader() = default; + +Status IcebergPositionDeleteSysTableV2Reader::prepare_split( + const format::SplitReadOptions& options) { + RETURN_IF_ERROR(close()); + RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + _current_range = options.current_range; + _has_split = true; + return _init_split(); +} + +Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.exec_timer); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(eos != nullptr); + DORIS_CHECK(block->columns() == _projected_columns.size()); + block->clear_column_data(_projected_columns.size()); + + if (*eos) { + return Status::OK(); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eos = true; + return Status::OK(); + } + if (!_has_split) { + *eos = true; + return Status::OK(); + } + + size_t read_rows = 0; + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _append_deletion_vector_block(block, &read_rows, eos); + } + + DORIS_CHECK(_position_reader != nullptr); + if (_batch_size > 0) { + _position_reader->set_batch_size(_batch_size); + } + + while (true) { + Block delete_block = _create_delete_block(); + bool position_reader_eof = false; + RETURN_IF_ERROR(_position_reader->get_block(&delete_block, &position_reader_eof)); + const size_t delete_rows = delete_block.rows(); + if (delete_rows > 0) { + RETURN_IF_ERROR( + _append_position_delete_block(block, delete_block, delete_rows, &read_rows)); + *eos = false; + return Status::OK(); + } + if (position_reader_eof) { + RETURN_IF_ERROR(close()); + *eos = true; + return Status::OK(); + } + } +} + +Status IcebergPositionDeleteSysTableV2Reader::close() { + Status close_status = Status::OK(); + if (_position_reader != nullptr) { + close_status = _position_reader->close(); + _position_reader.reset(); + } + auto base_status = format::TableReader::close(); + if (!base_status.ok() && close_status.ok()) { + close_status = std::move(base_status); + } + _iceberg_file_desc = nullptr; + _delete_file_desc = nullptr; + _read_columns.clear(); + _partition_value.reset(); + _next_dv_position.reset(); + _dv_positions = roaring::Roaring64Map(); + _has_split = false; + return close_status; +} + +std::string IcebergPositionDeleteSysTableV2Reader::debug_string() const { + std::ostringstream out; + out << "IcebergPositionDeleteSysTableV2Reader{base=" << format::TableReader::debug_string() + << ", has_split=" << _has_split << ", delete_file_kind=" + << (_delete_file_kind == DeleteFileKind::DELETION_VECTOR ? "DELETION_VECTOR" + : "POSITION_DELETE") + << ", read_column_count=" << _read_columns.size() + << ", dv_position_count=" << _dv_positions.cardinality() << "}"; + return out.str(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_split() { + if (_runtime_state == nullptr || _scanner_profile == nullptr || _scan_params == nullptr) { + return Status::InvalidArgument( + "invalid Iceberg position delete system table v2 reader context"); + } + if (_file_slot_descs == nullptr) { + return Status::InvalidArgument( + "Iceberg position delete system table v2 reader requires file slot descriptors"); + } + if (!_current_range.__isset.table_format_params || + !_current_range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("Iceberg position delete system table range misses params"); + } + + _iceberg_file_desc = &_current_range.table_format_params.iceberg_params; + if (!_iceberg_file_desc->__isset.delete_files || _iceberg_file_desc->delete_files.size() != 1) { + return Status::InternalError( + "Iceberg position delete system table range should contain exactly one delete " + "file"); + } + _delete_file_desc = _iceberg_file_desc->delete_files.data(); + if (is_iceberg_deletion_vector(*_delete_file_desc)) { + _delete_file_kind = DeleteFileKind::DELETION_VECTOR; + } else if (_delete_file_desc->__isset.content && + _delete_file_desc->content == kPositionDeleteContent) { + _delete_file_kind = DeleteFileKind::POSITION_DELETE; + } else if (!_delete_file_desc->__isset.content) { + return Status::InternalError( + "Iceberg position delete system table delete file misses content"); + } else { + return Status::InternalError( + "Iceberg position delete system table does not support delete file content {}", + _delete_file_desc->content); + } + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _init_deletion_vector_reader(); + } + return _init_position_delete_reader(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { + if (!_delete_file_desc->__isset.file_format) { + return Status::InternalError("Iceberg position delete file misses file format"); + } + format::FileFormat file_format; + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_PARQUET) { + file_format = format::FileFormat::PARQUET; + } else if (_delete_file_desc->file_format == TFileFormatType::FORMAT_ORC) { + file_format = format::FileFormat::ORC; + } else { + return Status::NotSupported( + "Iceberg position delete system table v2 reader only supports Parquet and ORC " + "delete files, file_format={}", + _delete_file_desc->file_format); + } + + const bool read_row = _output_column_requested(kRowColumn); + _init_read_columns(read_row); + std::vector projected_columns; + RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + + _position_reader = std::make_unique(); + RETURN_IF_ERROR(_position_reader->init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = file_format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::type::NONE, + .condition_cache_digest = 0, + })); + // Keep standalone-reader defaults for scanner-only fields that may be added to + // SplitReadOptions. + auto split_options = format::SplitReadOptions {}; + split_options.partition_values = {}; + split_options.partition_prune_conjuncts = {}; + split_options.cache = nullptr; + split_options.current_range = _current_range; + split_options.current_split_format = file_format; + split_options.global_rowid_context = std::nullopt; + RETURN_IF_ERROR(_position_reader->prepare_split(split_options)); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_deletion_vector_reader() { + if (!_delete_file_desc->__isset.referenced_data_file_path || + _delete_file_desc->referenced_data_file_path.empty()) { + return Status::InternalError("Iceberg deletion vector misses referenced data file path"); + } + if (_io_ctx == nullptr) { + return Status::InvalidArgument( + "Iceberg position delete system table v2 reader requires IO context"); + } + + IcebergDeleteFileReaderOptions options; + options.state = _runtime_state; + options.profile = _scanner_profile; + options.scan_params = _scan_params; + options.io_ctx = _io_ctx.get(); + if (_current_range.__isset.fs_name) { + options.fs_name = &_current_range.fs_name; + } + + _dv_positions = roaring::Roaring64Map(); + RETURN_IF_ERROR(read_iceberg_deletion_vector(*_delete_file_desc, options, &_dv_positions)); + _next_dv_position.emplace(_dv_positions.begin()); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_position_delete_block( + Block* output_block, const Block& delete_block, size_t delete_rows, size_t* appended_rows) { + auto columns_guard = output_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = output_block->get_name_to_pos_map(); + + for (size_t row = 0; row < delete_rows; ++row) { + for (const auto* slot : *_file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, &delete_block, row, 0)); + } + } + check_output_columns_aligned(columns); + *appended_rows = delete_rows; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_deletion_vector_block(Block* block, + size_t* read_rows, + bool* eof) { + const size_t batch_size = std::max( + _batch_size > 0 ? _batch_size + : (_runtime_state == nullptr + ? static_cast(102400) + : static_cast(_runtime_state->batch_size())), + 1); + if (!_next_dv_position.has_value() || *_next_dv_position == _dv_positions.end()) { + *read_rows = 0; + *eof = true; + RETURN_IF_ERROR(close()); + return Status::OK(); + } + + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = block->get_name_to_pos_map(); + + size_t rows = 0; + while (rows < batch_size && *_next_dv_position != _dv_positions.end()) { + const uint64_t dv_pos = **_next_dv_position; + for (const auto* slot : *_file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, nullptr, 0, dv_pos)); + } + ++(*_next_dv_position); + ++rows; + } + check_output_columns_aligned(columns); + *read_rows = rows; + _record_scan_rows(rows); + // FileScannerV2 treats eof=true as "advance to the next split" without returning the + // current block. Keep eof false after appending rows and report EOF on the next empty call. + *eof = false; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_sys_column(MutableColumnPtr& column, + const SlotDescriptor& slot, + const Block* delete_block, + size_t source_row, + uint64_t dv_pos) { + const std::string& name = slot.col_name(); + if (name == kFilePathColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_string(column, _delete_file_desc->referenced_data_file_path); + return Status::OK(); + } + const auto* source_column = get_column(*delete_block, kFilePathColumn); + const auto* path_column = get_string_column(source_column); + if (path_column == nullptr || !block_has_row(*delete_block, source_row) || + column_is_null_at(source_column, source_row)) { + return Status::InternalError("Iceberg position delete file_path column is missing"); + } + parquet_utils::insert_string(column, path_column->get_data_at(source_row).to_string()); + return Status::OK(); + } + + if (name == kPosColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_int64(column, cast_set(dv_pos)); + return Status::OK(); + } + const auto* source_column = get_column(*delete_block, kPosColumn); + const auto* pos_column = get_int64_column(source_column); + if (pos_column == nullptr || !block_has_row(*delete_block, source_row) || + column_is_null_at(source_column, source_row)) { + return Status::InternalError("Iceberg position delete pos column is missing"); + } + parquet_utils::insert_int64(column, pos_column->get_element(source_row)); + return Status::OK(); + } + + if (name == kRowColumn) { + if (delete_block != nullptr) { + auto row_pos = delete_block->get_position_by_name(kRowColumn); + if (row_pos >= 0) { + auto row_column = delete_block->get_by_position(row_pos) + .column->convert_to_full_column_if_const(); + if (source_row < row_column->size()) { + column->insert_from(*row_column, source_row); + return Status::OK(); + } + } + } + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (name == kPartitionColumn) { + return _append_partition_column(column, slot); + } + + if (name == kSpecIdColumn) { + if (_iceberg_file_desc->__isset.partition_spec_id) { + parquet_utils::insert_int32(column, + cast_set(_iceberg_file_desc->partition_spec_id)); + } else { + parquet_utils::insert_null(column); + } + return Status::OK(); + } + + if (name == kDeleteFilePathColumn) { + parquet_utils::insert_string(column, _delete_file_output_path()); + return Status::OK(); + } + + if (name == kContentOffsetColumn) { + const int64_t* value = _delete_file_desc->__isset.content_offset + ? &_delete_file_desc->content_offset + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + if (name == kContentSizeInBytesColumn) { + const int64_t* value = _delete_file_desc->__isset.content_size_in_bytes + ? &_delete_file_desc->content_size_in_bytes + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + return Status::InternalError("Unknown Iceberg position delete system table column: {}", name); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_partition_column(MutableColumnPtr& column, + const SlotDescriptor& slot) { + if (!_iceberg_file_desc->__isset.partition_data_json || + _iceberg_file_desc->partition_data_json.empty()) { + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (!_partition_value) { + auto partition_value = slot.get_data_type_ptr()->create_column(); + auto serde = slot.get_data_type_ptr()->get_serde(); + StringRef partition_data(_iceberg_file_desc->partition_data_json.data(), + _iceberg_file_desc->partition_data_json.size()); + DataTypeSerDe::FormatOptions options = DataTypeSerDe::get_default_format_options(); + RETURN_IF_ERROR(serde->from_string(partition_data, *partition_value, options)); + DORIS_CHECK(partition_value->size() == 1); + _partition_value = std::move(partition_value); + } + column->insert_from(*_partition_value, 0); + return Status::OK(); +} + +Block IcebergPositionDeleteSysTableV2Reader::_create_delete_block() const { + Block block; + for (const auto& column : _read_columns) { + block.insert(ColumnWithTypeAndName(column.type->create_column(), column.type, column.name)); + } + return block; +} + +bool IcebergPositionDeleteSysTableV2Reader::_output_column_requested( + const std::string& name) const { + return std::any_of(_file_slot_descs->begin(), _file_slot_descs->end(), + [&name](const SlotDescriptor* slot) { return slot->col_name() == name; }); +} + +void IcebergPositionDeleteSysTableV2Reader::_init_read_columns(bool read_row) { + _read_columns.clear(); + _read_columns.push_back({kFilePathColumn, make_nullable(std::make_shared())}); + _read_columns.push_back({kPosColumn, make_nullable(std::make_shared())}); + if (read_row) { + for (const auto* slot : *_file_slot_descs) { + if (slot->col_name() == kRowColumn) { + _read_columns.push_back({kRowColumn, slot->get_data_type_ptr()}); + break; + } + } + } +} + +Status IcebergPositionDeleteSysTableV2Reader::_build_delete_file_projected_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + if (column.name == kRowColumn) { + auto it = std::ranges::find_if(_projected_columns, [](const ColumnDefinition& field) { + return field.name == kRowColumn; + }); + if (it == _projected_columns.end()) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + columns->push_back(*it); + columns->back().type = column.type; + set_iceberg_delete_field_id(&columns->back()); + continue; + } + auto field = build_delete_file_column(column.name, column.type); + set_iceberg_delete_field_id(&field); + columns->push_back(std::move(field)); + } + return Status::OK(); +} + +const std::string& IcebergPositionDeleteSysTableV2Reader::_delete_file_output_path() const { + if (_delete_file_desc->__isset.original_path && !_delete_file_desc->original_path.empty()) { + return _delete_file_desc->original_path; + } + return _delete_file_desc->path; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h new file mode 100644 index 00000000000000..9c802e726053cb --- /dev/null +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "roaring/roaring64map.hh" + +namespace doris { +class SlotDescriptor; +} // namespace doris + +namespace doris::format::iceberg { + +class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { +public: + ~IcebergPositionDeleteSysTableV2Reader() override; + + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + Status close() override; + std::string debug_string() const override; + +private: + enum class DeleteFileKind { + POSITION_DELETE, + DELETION_VECTOR, + }; + + struct ReadColumn { + std::string name; + DataTypePtr type; + }; + + Status _init_split(); + Status _init_position_delete_reader(); + Status _init_deletion_vector_reader(); + Status _append_position_delete_block(Block* output_block, const Block& delete_block, + size_t delete_rows, size_t* appended_rows); + Status _append_deletion_vector_block(Block* block, size_t* read_rows, bool* eof); + Status _append_sys_column(MutableColumnPtr& column, const SlotDescriptor& slot, + const Block* delete_block, size_t source_row, uint64_t dv_pos); + Status _append_partition_column(MutableColumnPtr& column, const SlotDescriptor& slot); + Block _create_delete_block() const; + bool _output_column_requested(const std::string& name) const; + void _init_read_columns(bool read_row); + Status _build_delete_file_projected_columns(std::vector* columns) const; + const std::string& _delete_file_output_path() const; + + TFileRangeDesc _current_range; + const TIcebergFileDesc* _iceberg_file_desc = nullptr; + const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; + DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; + std::unique_ptr _position_reader; + std::vector _read_columns; + ColumnPtr _partition_value; + roaring::Roaring64Map _dv_positions; + std::optional _next_dv_position; + bool _has_split = false; +}; + +} // namespace doris::format::iceberg diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 586d307fe32edb..5f479a8843c830 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -48,6 +48,9 @@ namespace doris { namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + TFileRangeDesc range_with_format(std::string table_format, TFileFormatType::type format_type) { TFileRangeDesc range; range.__set_format_type(format_type); @@ -59,6 +62,14 @@ TFileRangeDesc range_with_format(std::string table_format, TFileFormatType::type return range; } +TFileRangeDesc iceberg_position_deletes_range(TFileFormatType::type format_type, int content) { + auto range = range_with_format("iceberg", format_type); + TIcebergFileDesc iceberg_params; + iceberg_params.__set_content(content); + range.table_format_params.__set_iceberg_params(std::move(iceberg_params)); + return range; +} + TFileRangeDesc hudi_range_with_delta_logs() { auto range = range_with_format("hudi", TFileFormatType::FORMAT_PARQUET); THudiFileDesc hudi_params; @@ -315,6 +326,27 @@ TEST(FileScannerV2Test, ConditionCacheDigestIncludesRuntimeFilterPayload) { 0); } +// Scenario: Iceberg position-delete system table splits use FileScannerV2 for both native delete +// formats and V3 deletion vectors. Avro remains unsupported and is rejected by FE before routing. +TEST(FileScannerV2Test, IcebergPositionDeletesSupportNativeFormats) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + const auto parquet_position_delete = iceberg_position_deletes_range( + TFileFormatType::FORMAT_PARQUET, kIcebergPositionDeleteContent); + const auto parquet_deletion_vector = iceberg_position_deletes_range( + TFileFormatType::FORMAT_PARQUET, kIcebergDeletionVectorContent); + const auto orc_position_delete = iceberg_position_deletes_range(TFileFormatType::FORMAT_ORC, + kIcebergPositionDeleteContent); + const auto avro_position_delete = iceberg_position_deletes_range(TFileFormatType::FORMAT_AVRO, + kIcebergPositionDeleteContent); + + EXPECT_TRUE(FileScannerV2::is_supported(params, parquet_position_delete)); + EXPECT_TRUE(FileScannerV2::is_supported(params, parquet_deletion_vector)); + EXPECT_TRUE(FileScannerV2::is_supported(params, orc_position_delete)); + EXPECT_FALSE(FileScannerV2::is_supported(params, avro_position_delete)); +} + TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesOnly) { TQueryOptions query_options; TFileScanRangeParams params; diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..bff48b9a6941d3 --- /dev/null +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,608 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { + +class ProfileTrackingReader final : public GenericReader { +public: + int collect_calls = 0; + + Status get_next_block(Block* /*block*/, size_t* /*read_rows*/, bool* /*eof*/) override { + return Status::OK(); + } + +protected: + void _collect_profile_before_close() override { ++collect_calls; } +}; + +SlotDescriptor* make_slot(ObjectPool* pool, int id, std::string name, DataTypePtr type) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_columnPos(id); + slot_desc.__set_byteOffset(0); + slot_desc.__set_nullIndicatorByte(id / 8); + slot_desc.__set_nullIndicatorBit(id % 8); + slot_desc.__set_slotIdx(id); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_colName(std::move(name)); + return pool->add(new SlotDescriptor(slot_desc)); +} + +Block make_output_block(const std::vector& slots) { + Block block; + for (const auto* slot : slots) { + auto type = slot->get_data_type_ptr(); + block.insert(ColumnWithTypeAndName(type->create_column(), type, slot->col_name())); + } + return block; +} + +const IColumn& nested_column(const Block& block, const std::string& name) { + const auto position = block.get_position_by_name(name); + DORIS_CHECK(position >= 0); + const auto& column = *block.get_by_position(position).column; + if (const auto* nullable = check_and_get_column(&column)) { + return nullable->get_nested_column(); + } + return column; +} + +bool is_null_at(const Block& block, const std::string& name, size_t row) { + const auto position = block.get_position_by_name(name); + DORIS_CHECK(position >= 0); + const auto* nullable = + check_and_get_column(block.get_by_position(position).column.get()); + DORIS_CHECK(nullable != nullptr); + return nullable->is_null_at(row); +} + +std::string string_at(const Block& block, const std::string& name, size_t row) { + const auto* column = check_and_get_column(&nested_column(block, name)); + DORIS_CHECK(column != nullptr); + return column->get_data_at(row).to_string(); +} + +Int64 int_at(const Block& block, const std::string& name, size_t row) { + return nested_column(block, name).get_int(row); +} + +Int64 struct_int_at(const Block& block, const std::string& name, size_t child_index, size_t row) { + const auto& struct_column = assert_cast(nested_column(block, name)); + const auto& child = assert_cast(struct_column.get_column(child_index)); + DORIS_CHECK(!child.is_null_at(row)); + return child.get_nested_column().get_int(row); +} + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +} // namespace + +TEST(IcebergPositionDeleteSysTableReaderTest, UsesScannerIOContext) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + io::FileReaderStats file_reader_stats; + scanner_io_ctx->file_reader_stats = &file_reader_stats; + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + + EXPECT_EQ(scanner_io_ctx.get(), reader._io_ctx.get()); + EXPECT_EQ(&file_reader_stats, reader._io_ctx->file_reader_stats); + scanner_io_ctx->should_stop = true; + EXPECT_TRUE(reader._io_ctx->should_stop); + + EXPECT_TRUE(reader.count_read_rows()); + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + EXPECT_FALSE(reader.count_read_rows()); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, ForwardsProfileCollectionToNestedReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + auto nested_reader = std::make_unique(); + auto* nested_reader_ptr = nested_reader.get(); + reader._position_reader = std::move(nested_reader); + + reader.collect_profile_before_close(); + reader.collect_profile_before_close(); + + EXPECT_EQ(1, nested_reader_ptr->collect_calls); + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + reader._partition_value = std::make_shared()->create_column(); + ASSERT_TRUE(reader.close().ok()); + EXPECT_TRUE(reader._dv_positions.isEmpty()); + EXPECT_FALSE(reader._next_dv_position.has_value()); + EXPECT_EQ(nullptr, reader._partition_value.get()); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, StopsBeforeExpandingDeletionVector) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + scanner_io_ctx->should_stop = true; + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + Block block; + size_t read_rows = 1; + bool eof = false; + + ASSERT_TRUE(reader.get_next_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(0, read_rows); + EXPECT_TRUE(eof); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, ValidatesRangeAndDeleteFileMetadata) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + TFileRangeDesc empty_range; + IcebergPositionDeleteSysTableReader invalid_context_reader( + file_slot_descs, &state, &profile, empty_range, ¶ms, nullptr, nullptr); + auto status = invalid_context_reader.init_reader(); + EXPECT_TRUE(status.is()) << status; + + IcebergPositionDeleteSysTableReader missing_params_reader( + file_slot_descs, &state, &profile, empty_range, ¶ms, scanner_io_ctx, nullptr); + status = missing_params_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("range misses params")); + + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(TIcebergFileDesc()); + TFileRangeDesc no_delete_file_range; + no_delete_file_range.__set_table_format_params(std::move(table_format_desc)); + IcebergPositionDeleteSysTableReader no_delete_file_reader(file_slot_descs, &state, &profile, + no_delete_file_range, ¶ms, + scanner_io_ctx, nullptr); + status = no_delete_file_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("exactly one delete file")); + + TIcebergDeleteFileDesc missing_content; + auto missing_content_range = range_with_delete_file(missing_content); + IcebergPositionDeleteSysTableReader missing_content_reader(file_slot_descs, &state, &profile, + missing_content_range, ¶ms, + scanner_io_ctx, nullptr); + status = missing_content_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses content")); + + TIcebergDeleteFileDesc equality_delete; + equality_delete.__set_content(2); + auto equality_delete_range = range_with_delete_file(equality_delete); + IcebergPositionDeleteSysTableReader equality_delete_reader(file_slot_descs, &state, &profile, + equality_delete_range, ¶ms, + scanner_io_ctx, nullptr); + status = equality_delete_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("does not support delete file content 2")); + + TIcebergDeleteFileDesc missing_format; + missing_format.__set_content(1); + auto missing_format_range = range_with_delete_file(missing_format); + IcebergPositionDeleteSysTableReader missing_format_reader(file_slot_descs, &state, &profile, + missing_format_range, ¶ms, + scanner_io_ctx, nullptr); + status = missing_format_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses file format")); + + TIcebergDeleteFileDesc unsupported_format; + unsupported_format.__set_content(1); + unsupported_format.__set_file_format(TFileFormatType::FORMAT_CSV_PLAIN); + auto unsupported_format_range = range_with_delete_file(unsupported_format); + IcebergPositionDeleteSysTableReader unsupported_format_reader(file_slot_descs, &state, &profile, + unsupported_format_range, ¶ms, + scanner_io_ctx, nullptr); + status = unsupported_format_reader.init_reader(); + EXPECT_TRUE(status.is()) << status; + + TIcebergDeleteFileDesc invalid_dv; + invalid_dv.__set_content(3); + auto invalid_dv_range = range_with_delete_file(invalid_dv); + IcebergPositionDeleteSysTableReader invalid_dv_reader( + file_slot_descs, &state, &profile, invalid_dv_range, ¶ms, scanner_io_ctx, nullptr); + status = invalid_dv_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses referenced data file path")); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsDeletionVectorMetadataAndCachesPartition) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + std::vector slots { + make_slot(&pool, 0, "file_path", nullable_string), + make_slot(&pool, 1, "pos", nullable_int64), + make_slot(&pool, 2, "row", nullable_int32), + make_slot(&pool, 3, "partition", partition_type), + make_slot(&pool, 4, "spec_id", nullable_int32), + make_slot(&pool, 5, "delete_file_path", nullable_string), + make_slot(&pool, 6, "content_offset", nullable_int64), + make_slot(&pool, 7, "content_size_in_bytes", nullable_int64), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_partition_spec_id(7); + iceberg_desc.__set_partition_data_json(R"({"p":42})"); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/physical-delete.puffin"); + delete_file.__set_original_path("s3://bucket/delete.puffin"); + delete_file.__set_referenced_data_file_path("s3://bucket/data.parquet"); + delete_file.__set_content_offset(12); + delete_file.__set_content_size_in_bytes(34); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {5}); + reader._dv_positions.add(uint64_t {9}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_FALSE(eof); + ASSERT_EQ(1, block.rows()); + ColumnPtr cached_partition = reader._partition_value; + ASSERT_NE(nullptr, cached_partition.get()); + EXPECT_EQ("s3://bucket/data.parquet", string_at(block, "file_path", 0)); + EXPECT_EQ(5, int_at(block, "pos", 0)); + EXPECT_TRUE(is_null_at(block, "row", 0)); + EXPECT_FALSE(is_null_at(block, "partition", 0)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 0)); + EXPECT_EQ(7, int_at(block, "spec_id", 0)); + EXPECT_EQ("s3://bucket/delete.puffin", string_at(block, "delete_file_path", 0)); + EXPECT_EQ(12, int_at(block, "content_offset", 0)); + EXPECT_EQ(34, int_at(block, "content_size_in_bytes", 0)); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + ASSERT_EQ(2, block.rows()); + EXPECT_EQ(cached_partition.get(), reader._partition_value.get()); + EXPECT_EQ(9, int_at(block, "pos", 1)); + EXPECT_FALSE(is_null_at(block, "partition", 1)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 1)); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(0, read_rows); + EXPECT_TRUE(eof); + + reader._dv_positions = roaring::Roaring64Map(); + reader._dv_positions.add(uint64_t {13}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + std::vector partial_slots {slots[0], slots[1]}; + Block partial_block = make_output_block(partial_slots); + ASSERT_TRUE(reader._append_deletion_vector_block(&partial_block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + EXPECT_EQ(13, int_at(partial_block, "pos", 0)); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsPositionDeleteRowsAndValidatesColumns) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector slots { + make_slot(&pool, 0, "file_path", nullable_string), + make_slot(&pool, 1, "pos", nullable_int64), + make_slot(&pool, 2, "row", nullable_int32), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + TIcebergFileDesc iceberg_desc; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/delete.parquet"); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::POSITION_DELETE; + + reader.set_batch_size(17); + EXPECT_EQ(17, reader.get_batch_size()); + EXPECT_TRUE(reader._output_column_requested("row")); + EXPECT_FALSE(reader._output_column_requested("missing")); + reader._init_read_columns(true); + ASSERT_EQ(3, reader._read_columns.size()); + EXPECT_EQ("row", reader._read_columns.back().name); + + std::unordered_map name_to_type; + std::unordered_set missing_column_names; + ASSERT_TRUE(reader.get_columns(&name_to_type, &missing_column_names).ok()); + EXPECT_TRUE(missing_column_names.empty()); + EXPECT_EQ(3, name_to_type.size()); + + Block delete_block = reader._create_delete_block(); + { + auto columns_guard = delete_block.mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + parquet_utils::insert_string(columns[0], "s3://bucket/data.parquet"); + parquet_utils::insert_int64(columns[1], 19); + parquet_utils::insert_int32(columns[2], 23); + } + Block output_block = make_output_block(slots); + size_t appended_rows = 0; + ASSERT_TRUE(reader._append_position_delete_block(&output_block, delete_block, 1, &appended_rows) + .ok()); + EXPECT_EQ(1, appended_rows); + ASSERT_EQ(1, output_block.rows()); + EXPECT_EQ("s3://bucket/data.parquet", string_at(output_block, "file_path", 0)); + EXPECT_EQ(19, int_at(output_block, "pos", 0)); + EXPECT_EQ(23, int_at(output_block, "row", 0)); + + Block empty_delete_block = reader._create_delete_block(); + auto path_column = nullable_string->create_column(); + auto status = reader._append_sys_column(path_column, *slots[0], &empty_delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("file_path column is missing")); + auto pos_column = nullable_int64->create_column(); + status = reader._append_sys_column(pos_column, *slots[1], &empty_delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("pos column is missing")); + auto row_column = nullable_int32->create_column(); + ASSERT_TRUE(reader._append_sys_column(row_column, *slots[2], &empty_delete_block, 0, 0).ok()); + ASSERT_EQ(1, row_column->size()); + + Block missing_columns; + path_column = nullable_string->create_column(); + status = reader._append_sys_column(path_column, *slots[0], &missing_columns, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("file_path column is missing")); + pos_column = nullable_int64->create_column(); + status = reader._append_sys_column(pos_column, *slots[1], &missing_columns, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("pos column is missing")); + + std::vector partial_slots {slots[0], slots[1]}; + Block partial_output_block = make_output_block(partial_slots); + appended_rows = 0; + ASSERT_TRUE(reader._append_position_delete_block(&partial_output_block, delete_block, 1, + &appended_rows) + .ok()); + EXPECT_EQ(1, appended_rows); + EXPECT_EQ(1, partial_output_block.rows()); + + auto* unknown_slot = make_slot(&pool, 3, "unknown", nullable_string); + auto unknown_column = nullable_string->create_column(); + status = reader._append_sys_column(unknown_column, *unknown_slot, &delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("Unknown Iceberg")); + + Block unused_block; + size_t read_rows = 0; + bool eof = false; + status = reader.get_next_block(&unused_block, &read_rows, &eof); + EXPECT_NE(std::string::npos, status.to_string().find("reader is not initialized")); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsNullMetadataAndUsesDeletePathFallback) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + std::vector slots { + make_slot(&pool, 0, "partition", partition_type), + make_slot(&pool, 1, "spec_id", nullable_int32), + make_slot(&pool, 2, "delete_file_path", nullable_string), + make_slot(&pool, 3, "content_offset", nullable_int64), + make_slot(&pool, 4, "content_size_in_bytes", nullable_int64), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + TIcebergFileDesc iceberg_desc; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/fallback-delete-file"); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + EXPECT_TRUE(is_null_at(block, "partition", 0)); + EXPECT_TRUE(is_null_at(block, "spec_id", 0)); + EXPECT_EQ("/fallback-delete-file", string_at(block, "delete_file_path", 0)); + EXPECT_TRUE(is_null_at(block, "content_offset", 0)); + EXPECT_TRUE(is_null_at(block, "content_size_in_bytes", 0)); + + std::vector empty_slots; + IcebergPositionDeleteSysTableReader empty_reader(empty_slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + empty_reader._dv_positions.add(uint64_t {2}); + empty_reader._next_dv_position.emplace(empty_reader._dv_positions.begin()); + Block empty_block; + ASSERT_TRUE(empty_reader._append_deletion_vector_block(&empty_block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) { + io::FileReaderStats file_reader_stats; + auto scanner_io_ctx = std::make_shared(); + scanner_io_ctx->file_reader_stats = &file_reader_stats; + std::vector file_slot_descs; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._io_ctx = scanner_io_ctx; + reader._file_slot_descs = &file_slot_descs; + reader._batch_size = 2; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block; + size_t read_rows = 0; + bool eof = true; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(2, read_rows); + EXPECT_FALSE(eof); + EXPECT_EQ(2, file_reader_stats.read_rows); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_FALSE(eof); + EXPECT_EQ(3, file_reader_stats.read_rows); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, CachesAndClearsPartitionValue) { + ObjectPool pool; + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + auto* partition_slot = make_slot(&pool, 0, "partition", partition_type); + + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_partition_data_json(R"({"p":42})"); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._iceberg_file_desc = &iceberg_desc; + auto partition_column = partition_type->create_column(); + ASSERT_TRUE(reader._append_partition_column(partition_column, *partition_slot).ok()); + ColumnPtr cached_partition = reader._partition_value; + ASSERT_NE(nullptr, cached_partition.get()); + + ASSERT_TRUE(reader._append_partition_column(partition_column, *partition_slot).ok()); + EXPECT_EQ(cached_partition.get(), reader._partition_value.get()); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(partition_column), partition_type, "partition")); + ASSERT_EQ(2, block.rows()); + EXPECT_FALSE(is_null_at(block, "partition", 0)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 0)); + EXPECT_FALSE(is_null_at(block, "partition", 1)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 1)); + + reader._has_split = true; + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + ASSERT_TRUE(reader.close().ok()); + EXPECT_EQ(nullptr, reader._iceberg_file_desc); + EXPECT_EQ(nullptr, reader._partition_value.get()); + EXPECT_TRUE(reader._dv_positions.isEmpty()); + EXPECT_FALSE(reader._next_dv_position.has_value()); + EXPECT_FALSE(reader._has_split); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ValidatesDeleteFileContentAfterBindingDescriptor) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc equality_delete; + equality_delete.__set_content(2); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(equality_delete); + + auto status = reader._init_split(); + EXPECT_NE(std::string::npos, status.to_string().find("does not support delete file content 2")); + ASSERT_NE(nullptr, reader._iceberg_file_desc); + ASSERT_NE(nullptr, reader._delete_file_desc); + EXPECT_EQ(reader._iceberg_file_desc->delete_files.data(), reader._delete_file_desc); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVector) { + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._io_ctx = std::make_shared(); + reader._io_ctx->should_stop = true; + + Block block; + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index d621b24d10253d..7dfdf6aed929bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -341,19 +341,6 @@ public Map getSupportedSysTables() { return IcebergSysTable.SUPPORTED_SYS_TABLES; } - @Override - public Optional findSysTable(String tableNameWithSysTableName) { - Optional sysTable = MTMVRelatedTableIf.super.findSysTable(tableNameWithSysTableName); - if (sysTable.isPresent()) { - return sysTable; - } - String sysTableName = SysTable.getTableNameWithSysTableName(tableNameWithSysTableName).second; - if (IcebergSysTable.POSITION_DELETES.equals(sysTableName)) { - return Optional.of(IcebergSysTable.UNSUPPORTED_POSITION_DELETES_TABLE); - } - return Optional.empty(); - } - @Override public boolean isView() { makeSureInitialized(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index ad90cb8fb4af6f..8c9c749c89426a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -46,12 +46,15 @@ import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TColumnCategory; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; @@ -64,9 +67,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.gson.JsonObject; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.BaseFileScanTask; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.BatchScan; +import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeleteFileIndex; @@ -76,11 +82,15 @@ import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; @@ -95,6 +105,8 @@ import org.apache.iceberg.mapping.MappedFields; import org.apache.iceberg.mapping.NameMapping; import org.apache.iceberg.mapping.NameMappingParser; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.ScanTaskUtil; import org.apache.iceberg.util.SerializationUtil; import org.apache.iceberg.util.TableScanUtil; @@ -102,6 +114,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -183,6 +196,18 @@ public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckCol scanContext, needCheckColumnPriv, sv); ExternalTable table = (ExternalTable) desc.getTable(); + initIcebergSource(table); + } + + public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, IcebergSysExternalTable sysExternalTable, + SessionVariable sv, ScanContext scanContext) { + super(id, desc, "ICEBERG_SCAN_NODE", StatisticalType.ICEBERG_SCAN_NODE, + scanContext, false, sv); + isSystemTable = true; + initIcebergSource(sysExternalTable); + } + + private void initIcebergSource(ExternalTable table) { if (table instanceof HMSExternalTable) { source = new IcebergHMSSource((HMSExternalTable) table, desc); } else if (table instanceof IcebergExternalTable || table instanceof IcebergSysExternalTable) { @@ -285,12 +310,19 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value()); TIcebergFileDesc fileDesc = new TIcebergFileDesc(); + if (isSystemTable && icebergSplit.isPositionDeleteSystemTableSplit()) { + setIcebergPositionDeleteSysTableParams(rangeDesc, icebergSplit, tableFormatFileDesc, fileDesc); + return; + } if (isSystemTable) { rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); tableFormatFileDesc.setTableLevelRowCount(-1); fileDesc.setSerializedSplit(icebergSplit.getSerializedSplit()); tableFormatFileDesc.setIcebergParams(fileDesc); rangeDesc.setTableFormatParams(tableFormatFileDesc); + rangeDesc.unsetColumnsFromPath(); + rangeDesc.unsetColumnsFromPathKeys(); + rangeDesc.unsetColumnsFromPathIsNull(); return; } // update for every split file format @@ -385,6 +417,41 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli rangeDesc.setTableFormatParams(tableFormatFileDesc); } + private void setIcebergPositionDeleteSysTableParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit, + TTableFormatFileDesc tableFormatFileDesc, TIcebergFileDesc fileDesc) { + rangeDesc.setFormatType(icebergSplit.getPositionDeleteFileFormat()); + tableFormatFileDesc.setTableLevelRowCount(-1); + fileDesc.setContent(icebergSplit.getPositionDeleteContent()); + + if (icebergSplit.getPartitionSpecId() != null) { + fileDesc.setPartitionSpecId(icebergSplit.getPartitionSpecId()); + } + if (icebergSplit.getPartitionDataJson() != null) { + fileDesc.setPartitionDataJson(icebergSplit.getPartitionDataJson()); + } + + TIcebergDeleteFileDesc deleteFileDesc = new TIcebergDeleteFileDesc(); + deleteFileDesc.setPath(rangeDesc.getPath()); + deleteFileDesc.setOriginalPath(icebergSplit.getPositionDeleteOriginalPath()); + deleteFileDesc.setFileFormat(icebergSplit.getPositionDeleteFileFormat()); + deleteFileDesc.setContent(icebergSplit.getPositionDeleteContent()); + if (icebergSplit.getPositionDeleteContentOffset() != null) { + deleteFileDesc.setContentOffset(icebergSplit.getPositionDeleteContentOffset()); + } + if (icebergSplit.getPositionDeleteContentSizeInBytes() != null) { + deleteFileDesc.setContentSizeInBytes(icebergSplit.getPositionDeleteContentSizeInBytes()); + } + if (icebergSplit.getPositionDeleteReferencedDataFilePath() != null) { + deleteFileDesc.setReferencedDataFilePath(icebergSplit.getPositionDeleteReferencedDataFilePath()); + } + fileDesc.setDeleteFiles(Lists.newArrayList(deleteFileDesc)); + tableFormatFileDesc.setIcebergParams(fileDesc); + rangeDesc.setTableFormatParams(tableFormatFileDesc); + rangeDesc.unsetColumnsFromPath(); + rangeDesc.unsetColumnsFromPathKeys(); + rangeDesc.unsetColumnsFromPathIsNull(); + } + @Override protected List getDeleteFiles(TFileRangeDesc rangeDesc) { List deleteFiles = new ArrayList<>(); @@ -456,6 +523,12 @@ public void createScanRangeLocations() throws UserException { @VisibleForTesting Map getBase64EncodedInitialDefaultsForScan() throws UserException { + if (isSystemTable) { + // System-table columns are derived from the metadata table schema. Some metadata + // tables, such as position_deletes, do not support Table.newScan(). Use the same + // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. + return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); + } TableScan tableScan = createTableScan(); Snapshot snapshot = tableScan.snapshot(); // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. @@ -640,11 +713,11 @@ private CloseableIterable splitFiles(TableScan scan) { return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(fileScanTaskList), targetSplitSize); } - private long determineTargetFileSplitSize(Iterable tasks) { + private long determineTargetFileSplitSize(Iterable> tasks) { long result = sessionVariable.getMaxInitialSplitSize(); long accumulatedTotalFileSize = 0; boolean exceedInitialThreshold = false; - for (FileScanTask task : tasks) { + for (ContentScanTask task : tasks) { accumulatedTotalFileSize += ScanTaskUtil.contentSizeInBytes(task.file()); if (!exceedInitialThreshold && accumulatedTotalFileSize >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) { @@ -658,6 +731,13 @@ private long determineTargetFileSplitSize(Iterable tasks) { return result; } + private long determinePositionDeleteTargetSplitSize(Iterable tasks) { + if (sessionVariable.getFileSplitSize() > 0) { + return sessionVariable.getFileSplitSize(); + } + return determineTargetFileSplitSize(tasks); + } + private CloseableIterable planFileScanTaskWithManifestCache(TableScan scan) throws IOException { // Get the snapshot from the scan; return empty if no snapshot exists Snapshot snapshot = scan.snapshot(); @@ -902,14 +982,148 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { return split; } - private Split createIcebergSysSplit(FileScanTask fileScanTask) { - long rowCount = fileScanTask.file() == null ? 1 : fileScanTask.file().recordCount(); + private Split createIcebergSysSplit(ScanTask scanTask) { + long rowCount = Math.max(scanTask.estimatedRowsCount(), 1L); + if (scanTask.isFileScanTask() && scanTask.asFileScanTask().file() != null) { + rowCount = Math.max(scanTask.asFileScanTask().file().recordCount(), 1L); + } IcebergSplit split = IcebergSplit.newSysTableSplit( - SerializationUtil.serializeToBase64(fileScanTask), rowCount); + SerializationUtil.serializeToBase64(scanTask), rowCount); split.setTableFormatType(TableFormatType.ICEBERG); return split; } + private Split createIcebergPositionDeleteSysSplit(PositionDeletesScanTask task) throws UserException { + DeleteFile deleteFile = task.file(); + String originalPath = deleteFile.path().toString(); + LocationPath locationPath = createLocationPathWithCache(originalPath); + IcebergSplit split = IcebergSplit.newPositionDeleteSysTableSplit( + locationPath, task.start(), task.length(), deleteFile.fileSizeInBytes(), + storagePropertiesMap, originalPath); + split.setTableFormatType(TableFormatType.ICEBERG); + split.setPositionDeleteFileFormat(getNativePositionDeleteFileFormat(deleteFile.format())); + split.setPositionDeleteOriginalPath(originalPath); + if (deleteFile.format() == FileFormat.PUFFIN) { + split.setPositionDeleteContent(IcebergDeleteFileFilter.DeletionVector.type()); + split.setPositionDeleteReferencedDataFilePath(deleteFile.referencedDataFile()); + split.setPositionDeleteContentOffset(deleteFile.contentOffset()); + split.setPositionDeleteContentSizeInBytes(deleteFile.contentSizeInBytes()); + } else { + split.setPositionDeleteContent(IcebergDeleteFileFilter.PositionDelete.type()); + } + + split.setPartitionSpecId(deleteFile.specId()); + PartitionSpec partitionSpec = icebergTable.specs().get(deleteFile.specId()); + Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s", + deleteFile.specId(), icebergTable.name()); + if (partitionSpec.isPartitioned() && deleteFile.partition() != null + && isPositionDeletesPartitionColumnRequested()) { + split.setPartitionDataJson(getPartitionDataObjectJson( + (PartitionData) deleteFile.partition(), partitionSpec, + getPositionDeletesOutputPartitionFields())); + } + return split; + } + + @SuppressWarnings("unchecked") + private Iterable splitPositionDeleteScanTask(PositionDeletesScanTask task) { + return ((SplittableScanTask) task).split(targetSplitSize); + } + + private TFileFormatType getNativePositionDeleteFileFormat(FileFormat fileFormat) { + if (fileFormat == FileFormat.PARQUET || fileFormat == FileFormat.PUFFIN) { + return TFileFormatType.FORMAT_PARQUET; + } else if (fileFormat == FileFormat.ORC) { + return TFileFormatType.FORMAT_ORC; + } + throw new UnsupportedOperationException( + "Unsupported Iceberg position delete file format: " + fileFormat); + } + + private List getPositionDeletesOutputPartitionFields() { + NestedField partitionField = icebergTable.schema().findField("partition"); + Preconditions.checkNotNull(partitionField, + "Partition field not found in Iceberg position_deletes metadata table schema"); + return partitionField.type().asNestedType().fields(); + } + + private boolean isPositionDeletesPartitionColumnRequested() { + return desc.getSlots().stream() + .anyMatch(slot -> "partition".equalsIgnoreCase(slot.getColumn().getName())); + } + + private String getPartitionDataObjectJson(PartitionData partitionData, PartitionSpec partitionSpec, + List outputPartitionFields) throws UserException { + List partitionTypes = partitionData.getPartitionType().asNestedType().fields(); + boolean enableMappingVarbinary = getEnableMappingVarbinary(); + for (int i = 0; i < partitionTypes.size(); i++) { + Type type = partitionTypes.get(i).type(); + if (partitionData.get(i) != null && (type.typeId() == Type.TypeID.BINARY + || type.typeId() == Type.TypeID.FIXED + || (type.typeId() == Type.TypeID.UUID && enableMappingVarbinary))) { + throw new UserException("Iceberg position_deletes cannot materialize non-null partition field '" + + partitionTypes.get(i).name() + "' of type " + type + + " without a binary-safe partition transport"); + } + } + List partitionValues = IcebergUtils.getPartitionValues( + partitionData, partitionSpec, sessionVariable.getTimeZone()); + Map partitionValueByFieldId = new HashMap<>(); + List fields = partitionSpec.fields(); + for (int i = 0; i < fields.size(); i++) { + partitionValueByFieldId.put(fields.get(i).fieldId(), + getPartitionJsonValue(partitionTypes.get(i).type(), partitionValues.get(i))); + } + JsonObject partitionJson = new JsonObject(); + for (NestedField outputPartitionField : outputPartitionFields) { + partitionJson.add(outputPartitionField.name(), + GsonUtils.GSON.toJsonTree(partitionValueByFieldId.get(outputPartitionField.fieldId()))); + } + return GsonUtils.GSON.toJson(partitionJson); + } + + private static Object getPartitionJsonValue(Type type, String partitionValue) { + if (partitionValue == null) { + return null; + } + switch (type.typeId()) { + case BOOLEAN: + return Boolean.parseBoolean(partitionValue); + case INTEGER: + return Integer.parseInt(partitionValue); + case LONG: + return Long.parseLong(partitionValue); + case FLOAT: + return Float.parseFloat(partitionValue); + case DOUBLE: + return Double.parseDouble(partitionValue); + case DECIMAL: + return new BigDecimal(partitionValue); + case STRING: + case UUID: + case DATE: + case TIME: + case TIMESTAMP: + return partitionValue; + default: + return partitionValue; + } + } + + @Override + protected TColumnCategory classifyColumn(SlotDescriptor slot, List partitionKeys) { + if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName())) { + return TColumnCategory.SYNTHESIZED; + } + if (slot.getColumn().getName().startsWith(Column.GLOBAL_ROWID_COL)) { + return TColumnCategory.SYNTHESIZED; + } + if (IcebergUtils.isIcebergRowLineageColumn(slot.getColumn())) { + return TColumnCategory.GENERATED; + } + return super.classifyColumn(slot, partitionKeys); + } + private List doGetSplits(int numBackends) throws UserException { if (isSystemTable) { return doGetSystemTableSplits(); @@ -964,6 +1178,9 @@ private List doGetSplits(int numBackends) throws UserException { } private List doGetSystemTableSplits() throws UserException { + if (isPositionDeletesSystemTable()) { + return doGetPositionDeletesSystemTableSplits(); + } List splits = new ArrayList<>(); TableScan scan = createTableScan(); long startTime = System.currentTimeMillis(); @@ -980,6 +1197,75 @@ private List doGetSystemTableSplits() throws UserException { return splits; } + private boolean isPositionDeletesSystemTable() { + TableIf targetTable = source.getTargetTable(); + return targetTable instanceof IcebergSysExternalTable + && "position_deletes".equalsIgnoreCase(((IcebergSysExternalTable) targetTable).getSysTableType()); + } + + private List doGetPositionDeletesSystemTableSplits() throws UserException { + checkPositionDeletesBackendCompatibility(backendPolicy.getBackends()); + List splits = new ArrayList<>(); + List positionDeleteTasks = new ArrayList<>(); + BatchScan scan = icebergTable.newBatchScan().metricsReporter(new IcebergMetricsReporter()); + + IcebergTableQueryInfo info = getSpecifiedSnapshot(); + if (info != null) { + if (info.getRef() != null) { + scan = scan.useRef(info.getRef()); + } else { + scan = scan.useSnapshot(info.getSnapshotId()); + } + } + + List expressions = new ArrayList<>(); + for (Expr conjunct : conjuncts) { + Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema()); + if (expression != null) { + expressions.add(expression); + } + } + for (Expression predicate : expressions) { + scan = scan.filter(predicate); + this.pushdownIcebergPredicates.add(predicate.toString()); + } + + long startTime = System.currentTimeMillis(); + scan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + try (CloseableIterable scanTasks = scan.planFiles()) { + for (ScanTask task : scanTasks) { + if (!(task instanceof PositionDeletesScanTask)) { + throw new UserException("Unexpected Iceberg position_deletes scan task: " + task); + } + positionDeleteTasks.add((PositionDeletesScanTask) task); + } + } catch (IOException e) { + throw new UserException(e.getMessage(), e); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } + } + targetSplitSize = determinePositionDeleteTargetSplitSize(positionDeleteTasks); + for (PositionDeletesScanTask task : positionDeleteTasks) { + for (PositionDeletesScanTask splitTask : splitPositionDeleteScanTask(task)) { + splits.add(createIcebergPositionDeleteSysSplit(splitTask)); + } + } + selectedPartitionNum = 0; + return splits; + } + + @VisibleForTesting + static void checkPositionDeletesBackendCompatibility(Iterable backends) throws UserException { + for (Backend backend : backends) { + if (backend.isSmoothUpgradeSrc()) { + throw new UserException("Iceberg position_deletes system table is unavailable while backend " + + backend.getId() + " is a smooth upgrade source"); + } + } + } + @Override public boolean isBatchMode() { if (isSystemTable) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java index 2987b12b1d52be..eeeff694b8ebce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java @@ -20,6 +20,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.FileSplit; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.thrift.TFileFormatType; import lombok.Data; import org.apache.iceberg.DeleteFile; @@ -55,6 +56,13 @@ public class IcebergSplit extends FileSplit { private String serializedSplit; // maybe mixed file format type in one table. so need record it for every split private FileFormat splitFileFormat; + private boolean positionDeleteSystemTableSplit = false; + private TFileFormatType positionDeleteFileFormat; + private int positionDeleteContent; + private String positionDeleteOriginalPath; + private String positionDeleteReferencedDataFilePath; + private Long positionDeleteContentOffset; + private Long positionDeleteContentSizeInBytes; // File path will be changed if the file is modified, so there's no need to get modification time. public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts, @@ -81,4 +89,13 @@ public static IcebergSplit newSysTableSplit(String serializedSplit, long rowCoun split.setSelfSplitWeight(Math.max(rowCount, 1L)); return split; } + + public static IcebergSplit newPositionDeleteSysTableSplit(LocationPath file, long start, long length, + long fileLength, Map config, String originalPath) { + IcebergSplit split = new IcebergSplit(file, start, length, fileLength, null, null, config, + Collections.emptyList(), originalPath); + split.setPositionDeleteSystemTableSplit(true); + split.setSelfSplitWeight(Math.max(length, 1L)); + return split; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java index 72a739e7b9a64b..5115c2f9ec6f95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java @@ -20,7 +20,6 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.iceberg.MetadataTableType; @@ -40,27 +39,20 @@ * @see org.apache.iceberg.MetadataTableType for all supported system table types */ public class IcebergSysTable extends NativeSysTable { - public static final String POSITION_DELETES = MetadataTableType.POSITION_DELETES.name().toLowerCase(Locale.ROOT); - /** * All supported Iceberg system tables. * Key is the system table name (e.g., "snapshots", "history"). */ public static final Map SUPPORTED_SYS_TABLES = Collections.unmodifiableMap( Arrays.stream(MetadataTableType.values()) - .filter(type -> type != MetadataTableType.POSITION_DELETES) - .map(type -> new IcebergSysTable(type.name().toLowerCase(Locale.ROOT), true)) + .map(type -> new IcebergSysTable(type.name().toLowerCase(Locale.ROOT))) .collect(Collectors.toMap(SysTable::getSysTableName, Function.identity()))); - public static final SysTable UNSUPPORTED_POSITION_DELETES_TABLE = - new IcebergSysTable(POSITION_DELETES, false); private final String tableName; - private final boolean supported; - private IcebergSysTable(String tableName, boolean supported) { + private IcebergSysTable(String tableName) { super(tableName); this.tableName = tableName; - this.supported = supported; } @Override @@ -70,9 +62,6 @@ public String getSysTableName() { @Override public ExternalTable createSysExternalTable(ExternalTable sourceTable) { - if (!supported) { - throw new AnalysisException("SysTable " + tableName + " is not supported yet"); - } if (!(sourceTable instanceof IcebergExternalTable)) { throw new IllegalArgumentException( "Expected IcebergExternalTable but got " + sourceTable.getClass().getSimpleName()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 87e2fb62e52b96..ec72503f7a674f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -19,11 +19,13 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.TableFormatType; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TIcebergDeleteFileDesc; @@ -31,6 +33,8 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -42,20 +46,29 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; private static class TestIcebergScanNode extends IcebergScanNode { + private final boolean enableMappingVarbinary; private TableScan tableScan; TestIcebergScanNode(SessionVariable sv) { + this(sv, false); + } + + TestIcebergScanNode(SessionVariable sv, boolean enableMappingVarbinary) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); + this.enableMappingVarbinary = enableMappingVarbinary; } void setTableScan(TableScan tableScan) { @@ -71,6 +84,11 @@ public TableScan createTableScan() { public boolean isBatchMode() { return false; } + + @Override + protected boolean getEnableMappingVarbinary() { + return enableMappingVarbinary; + } } @Test @@ -101,6 +119,30 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } + @Test + public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { + Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); + icebergTableField.setAccessible(true); + icebergTableField.set(node, systemTable); + Field isSystemTableField = IcebergScanNode.class.getDeclaredField("isSystemTable"); + isSystemTableField.setAccessible(true); + isSystemTableField.setBoolean(node, true); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + Mockito.verify(node, Mockito.never()).createTableScan(); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); @@ -212,4 +254,98 @@ public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exce .get(0); Assert.assertEquals(org.apache.doris.thrift.TFileFormatType.FORMAT_ORC, deleteFileDesc.getFileFormat()); } + + @Test + public void testPartitionDataJsonMatchesRenamedFieldById() throws Exception { + SessionVariable sv = new SessionVariable(); + TestIcebergScanNode node = new TestIcebergScanNode(sv); + + Schema schema = new Schema(Types.NestedField.required(1, "p", Types.IntegerType.get())); + PartitionSpec oldSpec = PartitionSpec.builderFor(schema).identity("p").build(); + PartitionData partitionData = new PartitionData(oldSpec.partitionType()); + partitionData.set(0, 10); + int partitionFieldId = oldSpec.fields().get(0).fieldId(); + List outputPartitionFields = Collections.singletonList( + Types.NestedField.optional(partitionFieldId, "p2", Types.IntegerType.get())); + + Method method = IcebergScanNode.class.getDeclaredMethod("getPartitionDataObjectJson", + PartitionData.class, PartitionSpec.class, List.class); + method.setAccessible(true); + + Assert.assertEquals("{\"p2\":10}", method.invoke(node, partitionData, oldSpec, outputPartitionFields)); + } + + @Test + public void testRejectBinaryPartitionValueWithoutBinarySafeTransport() throws Exception { + assertUnsupportedPositionDeletesPartitionValue( + Types.BinaryType.get(), ByteBuffer.wrap(new byte[] {0, (byte) 0xff}), false, "binary"); + assertUnsupportedPositionDeletesPartitionValue( + Types.FixedType.ofLength(2), ByteBuffer.wrap(new byte[] {0, (byte) 0xff}), false, "fixed[2]"); + } + + @Test + public void testRejectUuidPartitionValueWhenMappedToVarbinary() throws Exception { + assertUnsupportedPositionDeletesPartitionValue( + Types.UUIDType.get(), UUID.fromString("123e4567-e89b-12d3-a456-426614174000"), true, "uuid"); + } + + private void assertUnsupportedPositionDeletesPartitionValue( + org.apache.iceberg.types.Type type, Object value, boolean enableMappingVarbinary, + String expectedType) throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable(), enableMappingVarbinary); + Schema schema = new Schema(Types.NestedField.required(1, "p", type)); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("p").build(); + PartitionData partitionData = new PartitionData(spec.partitionType()); + partitionData.set(0, value); + + Method method = IcebergScanNode.class.getDeclaredMethod("getPartitionDataObjectJson", + PartitionData.class, PartitionSpec.class, List.class); + method.setAccessible(true); + try { + method.invoke(node, partitionData, spec, spec.partitionType().fields()); + Assert.fail("Binary partition values must not be silently materialized as NULL"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof UserException); + Assert.assertTrue(e.getCause().getMessage().contains("partition field 'p'")); + Assert.assertTrue(e.getCause().getMessage().contains(expectedType)); + } + } + + @Test + public void testRejectUnsupportedPositionDeleteFileFormat() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Method method = IcebergScanNode.class.getDeclaredMethod( + "getNativePositionDeleteFileFormat", FileFormat.class); + method.setAccessible(true); + + try { + method.invoke(node, FileFormat.AVRO); + Assert.fail("AVRO position delete files should be rejected explicitly"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("Unsupported Iceberg position delete file format: AVRO", + e.getCause().getMessage()); + } + } + + @Test + public void testRejectSmoothUpgradeSourceBackendForPositionDeletes() throws Exception { + Backend currentBackend = Mockito.mock(Backend.class); + Mockito.when(currentBackend.isSmoothUpgradeSrc()).thenReturn(false); + IcebergScanNode.checkPositionDeletesBackendCompatibility(Collections.singletonList(currentBackend)); + + Backend smoothUpgradeSource = Mockito.mock(Backend.class); + Mockito.when(smoothUpgradeSource.isSmoothUpgradeSrc()).thenReturn(true); + Mockito.when(smoothUpgradeSource.getId()).thenReturn(10001L); + List backends = new ArrayList<>(); + backends.add(currentBackend); + backends.add(smoothUpgradeSource); + + try { + IcebergScanNode.checkPositionDeletesBackendCompatibility(backends); + Assert.fail("smooth upgrade source backend should reject native position_deletes planning"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("backend 10001 is a smooth upgrade source")); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java index e6c5b0be328f99..d2627befd3734c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java @@ -21,7 +21,6 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; -import org.apache.doris.nereids.exceptions.AnalysisException; import mockit.Expectations; import mockit.Mocked; @@ -37,8 +36,8 @@ public class IcebergSysTableResolverTest { private IcebergExternalDatabase db; @Test - public void testSupportedSysTablesExcludePositionDeletes() { - Assertions.assertFalse(IcebergSysTable.SUPPORTED_SYS_TABLES.containsKey(IcebergSysTable.POSITION_DELETES)); + public void testSupportedSysTablesIncludePositionDeletes() { + Assertions.assertTrue(IcebergSysTable.SUPPORTED_SYS_TABLES.containsKey("position_deletes")); } @Test @@ -61,10 +60,14 @@ public void testResolveForPlanAndDescribeUseNativePath() throws Exception { } @Test - public void testPositionDeletesKeepsUnsupportedError() throws Exception { + public void testPositionDeletesUseNativePath() throws Exception { IcebergExternalTable sourceTable = newIcebergTable(); - Assertions.assertThrows(AnalysisException.class, () -> - SysTableResolver.resolveForPlan(sourceTable, "test_ctl", "test_db", "tbl$position_deletes")); + Optional plan = SysTableResolver.resolveForPlan( + sourceTable, "test_ctl", "test_db", "tbl$position_deletes"); + Assertions.assertTrue(plan.isPresent()); + Assertions.assertTrue(plan.get().isNative()); + Assertions.assertTrue(plan.get().getSysExternalTable() instanceof IcebergSysExternalTable); + Assertions.assertEquals("tbl$position_deletes", plan.get().getSysExternalTable().getName()); } private IcebergExternalTable newIcebergTable() throws Exception { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 47bc948f2291c1..59c158e6d2d2ab 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -297,6 +297,10 @@ struct TIcebergDeleteFileDesc { 6: optional i64 content_offset; 7: optional i64 content_size_in_bytes; 8: optional TFileFormatType file_format; + // Original Iceberg delete file path before Doris storage path normalization. + 9: optional string original_path; + // Referenced data file path. Required to materialize rows from deletion vectors. + 10: optional string referenced_data_file_path; } struct TIcebergFileDesc { diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out new file mode 100644 index 00000000000000..d7f3395f293c2a --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_rename_field -- +0 10 +1 10 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy new file mode 100644 index 00000000000000..dae16a9f3c7b66 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy @@ -0,0 +1,659 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_position_deletes_sys_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_position_deletes_sys_table" + String dbName = "position_deletes_sys_table_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + );""" + + spark_iceberg_multi """ + SET spark.sql.shuffle.partitions=1; + CREATE DATABASE IF NOT EXISTS demo.${dbName}; + DROP TABLE IF EXISTS demo.${dbName}.pd_unpartitioned; + CREATE TABLE demo.${dbName}.pd_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd') AS t(id, name); + DELETE FROM demo.${dbName}.pd_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_orc_unpartitioned; + CREATE TABLE demo.${dbName}.pd_orc_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.format.default'='orc', + 'write.delete.format.default'='orc', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_orc_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_orc_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_partitioned; + CREATE TABLE demo.${dbName}.pd_partitioned ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partitioned + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'a', '2026-06-26'), + (2, 'b', '2026-06-26'), + (3, 'c', '2026-06-27'), + (4, 'd', '2026-06-27') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_int_partitioned; + CREATE TABLE demo.${dbName}.pd_int_partitioned ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_int_partitioned + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_int_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_date_partitioned; + CREATE TABLE demo.${dbName}.pd_date_partitioned ( + id INT, + name STRING, + dt DATE + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_date_partitioned + SELECT /*+ COALESCE(1) */ id, name, CAST(dt AS DATE) FROM VALUES + (1, 'a', '2026-01-02'), + (2, 'b', '2026-01-02'), + (3, 'c', '2026-03-04') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_date_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_partition_evolution; + CREATE TABLE demo.${dbName}.pd_partition_evolution ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partition_evolution + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20), + (4, 'd', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_evolution WHERE id = 2; + ALTER TABLE demo.${dbName}.pd_partition_evolution ADD PARTITION FIELD bucket(1, id); + INSERT INTO demo.${dbName}.pd_partition_evolution + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (5, 'e', 10), + (6, 'f', 10) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_evolution WHERE id = 6; + DROP TABLE IF EXISTS demo.${dbName}.pd_partition_rename; + CREATE TABLE demo.${dbName}.pd_partition_rename ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partition_rename + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20), + (4, 'd', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_rename WHERE id = 2; + -- Replacing an identity transform with the same transform and an alias renames the + -- partition field while preserving its Iceberg field ID. + ALTER TABLE demo.${dbName}.pd_partition_rename REPLACE PARTITION FIELD p WITH p AS p2; + INSERT INTO demo.${dbName}.pd_partition_rename + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (5, 'e', 10), + (6, 'f', 10) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_rename WHERE id = 6; + DROP TABLE IF EXISTS demo.${dbName}.pd_v3_unpartitioned; + CREATE TABLE demo.${dbName}.pd_v3_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_v3_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_v3_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_v3_partitioned; + CREATE TABLE demo.${dbName}.pd_v3_partitioned ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='3', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_v3_partitioned + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'a', '2026-07-01'), + (2, 'b', '2026-07-01'), + (3, 'c', '2026-07-02') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_v3_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_schema_time_travel; + CREATE TABLE demo.${dbName}.pd_schema_time_travel ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_schema_time_travel + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_schema_time_travel WHERE id = 2; + ALTER TABLE demo.${dbName}.pd_schema_time_travel ADD COLUMN note STRING; + INSERT INTO demo.${dbName}.pd_schema_time_travel + SELECT /*+ COALESCE(1) */ id, name, note FROM VALUES + (4, 'd', 'new'), (5, 'e', 'new') AS t(id, name, note); + DELETE FROM demo.${dbName}.pd_schema_time_travel WHERE id = 4; + DROP TABLE IF EXISTS demo.${dbName}.pd_no_deletes; + CREATE TABLE demo.${dbName}.pd_no_deletes ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ('format-version'='2'); + INSERT INTO demo.${dbName}.pd_no_deletes VALUES (1, 'a'), (2, 'b'); + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + def assertPositionDeletesSchema = { String tableName, boolean partitioned, List extraColumns = [] -> + List> descRows = sql """desc ${tableName}\$position_deletes""" + List columns = descRows.collect { it[0].toString() } + List expectedColumns = ["file_path", "pos", "row"] + if (partitioned) { + expectedColumns.add("partition") + } + expectedColumns.addAll(["spec_id", "delete_file_path"]) + expectedColumns.addAll(extraColumns) + assertEquals(expectedColumns, columns) + } + + def countRows = { String query -> + List> rows = sql query + assertEquals(1, rows.size()) + return ((Number) rows[0][0]).longValue() + } + + def assertSparkDorisPositionDeletes = { String tableName, List columns -> + String columnList = columns.join(", ") + String orderBy = "file_path, pos, delete_file_path" + List> sparkRows = spark_iceberg """ + select ${columnList} + from demo.${dbName}.${tableName}.position_deletes + order by ${orderBy} + """ + List> dorisRows = sql """ + select ${columnList} + from ${tableName}\$position_deletes + order by ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertSparkDorisTableRows = { String tableName, List> expectedRows -> + spark_iceberg """REFRESH TABLE demo.${dbName}.${tableName}""" + List> sparkRows = spark_iceberg """ + select id, name, dt + from demo.${dbName}.${tableName} + order by id + """ + List> dorisRows = sql """ + select id, name, dt + from ${tableName} + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + assertEquals(expectedRows, dorisRows) + } + + List v3ExtraColumns = ["content_offset", "content_size_in_bytes"] + List commonCompareColumns = ["file_path", "pos", "spec_id", "delete_file_path"] + List v3CompareColumns = commonCompareColumns + v3ExtraColumns + + assertPositionDeletesSchema("pd_unpartitioned", false) + long unpartitionedCount = countRows("""select count(*) from pd_unpartitioned\$position_deletes""") + assertTrue(unpartitionedCount > 0) + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertEquals(unpartitionedCount, countRows("""select count(pos) from pd_unpartitioned\$position_deletes""")) + assertEquals(unpartitionedCount, (long) sql( + """select * from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select pos from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select delete_file_path, pos from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select file_path, pos, delete_file_path from pd_unpartitioned\$position_deletes where pos >= 0""").size()) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where spec_id = 0""")) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where delete_file_path is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_unpartitioned\$position_deletes where pos < 0""")) + List> unpartitionedRows = sql """select `row` from pd_unpartitioned\$position_deletes""" + assertEquals(unpartitionedCount, (long) unpartitionedRows.size()) + + assertPositionDeletesSchema("pd_orc_unpartitioned", false) + long orcUnpartitionedCount = countRows( + """select count(*) from pd_orc_unpartitioned\$position_deletes""") + assertTrue(orcUnpartitionedCount > 0) + assertSparkDorisPositionDeletes("pd_orc_unpartitioned", commonCompareColumns) + List> orcUnpartitionedRows = sql """select `row` from pd_orc_unpartitioned\$position_deletes""" + assertEquals(orcUnpartitionedCount, (long) orcUnpartitionedRows.size()) + assertTrue(orcUnpartitionedRows.every { it[0] == null }) + try { + sql """set file_split_size=1""" + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where pos >= 0""")) + assertEquals(orcUnpartitionedCount, countRows( + """select count(*) from pd_orc_unpartitioned\$position_deletes where pos >= 0""")) + } finally { + sql """unset variable file_split_size""" + } + assertEquals([[1, "a"], [3, "c"], [4, "d"]], sql("""select * from pd_unpartitioned order by id""")) + + assertPositionDeletesSchema("pd_partitioned", true) + long partitionedCount = countRows("""select count(*) from pd_partitioned\$position_deletes""") + assertTrue(partitionedCount > 0) + assertSparkDorisPositionDeletes("pd_partitioned", commonCompareColumns) + assertEquals(partitionedCount, countRows("""select count(pos) from pd_partitioned\$position_deletes""")) + assertEquals(partitionedCount, (long) sql( + """select * from pd_partitioned\$position_deletes""").size()) + assertEquals(partitionedCount, (long) sql( + """select file_path, pos, delete_file_path from pd_partitioned\$position_deletes where pos >= 0""").size()) + assertEquals(partitionedCount, countRows( + """select count(*) from pd_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_partitioned\$position_deletes where pos < 0""")) + List> partitionedRows = sql """select `row`, `partition` from pd_partitioned\$position_deletes""" + assertEquals(partitionedCount, (long) partitionedRows.size()) + assertTrue(partitionedRows.every { it[0] == null && it[1] != null }) + assertEquals([[1, "a", "2026-06-26"], [3, "c", "2026-06-27"], [4, "d", "2026-06-27"]], + sql("""select * from pd_partitioned order by id""")) + + assertPositionDeletesSchema("pd_int_partitioned", true) + long intPartitionedCount = countRows("""select count(*) from pd_int_partitioned\$position_deletes""") + assertTrue(intPartitionedCount > 0) + assertEquals(intPartitionedCount, countRows("""select count(pos) from pd_int_partitioned\$position_deletes""")) + assertEquals(intPartitionedCount, countRows( + """select count(*) from pd_int_partitioned\$position_deletes where `partition` is not null""")) + List> intPartitionedRows = sql """select `row`, `partition` from pd_int_partitioned\$position_deletes""" + assertEquals(intPartitionedCount, (long) intPartitionedRows.size()) + assertTrue(intPartitionedRows.every { it[0] == null && it[1] != null }) + assertTrue(intPartitionedRows.every { + String partitionValue = it[1].toString() + partitionValue.contains("\"p\":10") && !partitionValue.contains("\"p\":\"10\"") + }) + assertEquals([[1, "a", 10], [3, "c", 20]], sql("""select * from pd_int_partitioned order by id""")) + + // DATE-typed partition column. Iceberg serializes the partition value to an ISO date string + // (`2026-01-02`) which the BE parses back into the metadata table `partition` struct DATE field. + // Verifies the typed partition round-trip is not limited to STRING/INT columns. + assertPositionDeletesSchema("pd_date_partitioned", true) + long datePartitionedCount = countRows("""select count(*) from pd_date_partitioned\$position_deletes""") + assertEquals(1L, datePartitionedCount) + assertSparkDorisPositionDeletes("pd_date_partitioned", commonCompareColumns) + assertEquals(datePartitionedCount, countRows( + """select count(*) from pd_date_partitioned\$position_deletes where `partition` is not null""")) + List> datePartitionedRows = sql """ + select `row`, cast(`partition` as string) from pd_date_partitioned\$position_deletes + """ + assertEquals(datePartitionedCount, (long) datePartitionedRows.size()) + assertTrue(datePartitionedRows.every { it[0] == null && it[1] != null }) + // The deleted row (id=2) lives in partition dt=2026-01-02, so the partition value must render + // that exact date rather than a null / days-since-epoch integer / mangled string. + assertTrue(datePartitionedRows.every { it[1].toString().contains("2026-01-02") }) + assertEquals([[1, "a", "2026-01-02"], [3, "c", "2026-03-04"]], + sql("""select id, name, cast(dt as string) from pd_date_partitioned order by id""")) + + assertPositionDeletesSchema("pd_partition_evolution", true) + long partitionEvolutionCount = countRows("""select count(*) from pd_partition_evolution\$position_deletes""") + assertEquals(2L, partitionEvolutionCount) + assertSparkDorisPositionDeletes("pd_partition_evolution", commonCompareColumns) + assertEquals(partitionEvolutionCount, countRows( + """select count(pos) from pd_partition_evolution\$position_deletes""")) + assertEquals(partitionEvolutionCount, countRows( + """select count(*) from pd_partition_evolution\$position_deletes where `partition` is not null""")) + List> partitionEvolutionRows = sql """ + select spec_id, cast(`partition` as string) + from pd_partition_evolution\$position_deletes order by spec_id + """ + assertEquals(2, partitionEvolutionRows.size()) + assertEquals(0, ((Number) partitionEvolutionRows[0][0]).intValue()) + assertEquals(1, ((Number) partitionEvolutionRows[1][0]).intValue()) + String oldSpecPartition = partitionEvolutionRows[0][1].toString() + String newSpecPartition = partitionEvolutionRows[1][1].toString() + assertTrue(oldSpecPartition.contains("\"p\":10")) + assertTrue(oldSpecPartition.contains(":null")) + assertTrue(newSpecPartition.contains("\"p\":10")) + assertFalse(newSpecPartition.contains(":null")) + assertEquals([[1, "a", 10], [3, "c", 20], [4, "d", 20], [5, "e", 10]], + sql("""select * from pd_partition_evolution order by id""")) + + assertPositionDeletesSchema("pd_partition_rename", true) + long partitionRenameCount = countRows("""select count(*) from pd_partition_rename\$position_deletes""") + order_qt_partition_rename_field """ + select spec_id, struct_element(`partition`, 'p2') + from pd_partition_rename\$position_deletes + order by spec_id + """ + List> sparkPartitionRenameRows = spark_iceberg """ + select spec_id, partition.p2 + from demo.${dbName}.pd_partition_rename.position_deletes + order by spec_id + """ + List> dorisPartitionRenameRows = sql """ + select spec_id, struct_element(`partition`, 'p2') + from pd_partition_rename\$position_deletes + order by spec_id + """ + assertSparkDorisResultEquals(sparkPartitionRenameRows, dorisPartitionRenameRows) + assertEquals(partitionRenameCount, countRows(""" + select count(*) from pd_partition_rename\$position_deletes + where struct_element(`partition`, 'p2') = 10 + """)) + + assertPositionDeletesSchema("pd_v3_unpartitioned", false, v3ExtraColumns) + long v3UnpartitionedCount = countRows("""select count(*) from pd_v3_unpartitioned\$position_deletes""") + assertEquals(1L, v3UnpartitionedCount) + assertSparkDorisPositionDeletes("pd_v3_unpartitioned", v3CompareColumns) + assertEquals(v3UnpartitionedCount, countRows( + """select count(pos) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(content_offset) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(content_size_in_bytes) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, (long) sql( + """select file_path, pos, delete_file_path, content_offset, content_size_in_bytes + from pd_v3_unpartitioned\$position_deletes""").size()) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes where spec_id = 0""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes where delete_file_path is not null""")) + List> v3UnpartitionedRows = sql """select `row` from pd_v3_unpartitioned\$position_deletes""" + assertEquals(v3UnpartitionedCount, (long) v3UnpartitionedRows.size()) + assertTrue(v3UnpartitionedRows.every { it[0] == null }) + assertEquals([[1, "a"], [3, "c"]], sql("""select * from pd_v3_unpartitioned order by id""")) + + assertPositionDeletesSchema("pd_v3_partitioned", true, v3ExtraColumns) + long v3PartitionedCount = countRows("""select count(*) from pd_v3_partitioned\$position_deletes""") + assertEquals(1L, v3PartitionedCount) + assertSparkDorisPositionDeletes("pd_v3_partitioned", v3CompareColumns) + assertEquals(v3PartitionedCount, countRows( + """select count(pos) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, countRows( + """select count(content_offset) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, countRows( + """select count(content_size_in_bytes) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, (long) sql( + """select file_path, pos, delete_file_path, content_offset, content_size_in_bytes + from pd_v3_partitioned\$position_deletes""").size()) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where spec_id = 0""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where delete_file_path is not null""")) + List> v3PartitionedRows = sql """select `row`, `partition` from pd_v3_partitioned\$position_deletes""" + assertEquals(v3PartitionedCount, (long) v3PartitionedRows.size()) + assertTrue(v3PartitionedRows.every { it[0] == null && it[1] != null }) + assertEquals([[1, "a", "2026-07-01"], [3, "c", "2026-07-02"]], + sql("""select * from pd_v3_partitioned order by id""")) + + assertPositionDeletesSchema("pd_schema_time_travel", false) + long schemaChangeCount = countRows("""select count(*) from pd_schema_time_travel\$position_deletes""") + assertEquals(2L, schemaChangeCount) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where pos >= 0""")) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where spec_id = 0""")) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where delete_file_path is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_schema_time_travel\$position_deletes where pos < 0""")) + List> schemaChangeRows = sql """select `row` from pd_schema_time_travel\$position_deletes""" + assertEquals(schemaChangeCount, (long) schemaChangeRows.size()) + assertTrue(schemaChangeRows.every { it[0] == null }) + assertEquals([[1, "a", null], [3, "c", null], [5, "e", "new"]], + sql("""select * from pd_schema_time_travel order by id""")) + + List> schemaChangeSnapshots = sql """ + select snapshot_id from pd_schema_time_travel\$snapshots order by committed_at + """ + assertEquals(4, schemaChangeSnapshots.size()) + Object afterInsertSnapshot = schemaChangeSnapshots.get(0)[0] + Object afterFirstDeleteSnapshot = schemaChangeSnapshots.get(1)[0] + Object afterSecondDeleteSnapshot = schemaChangeSnapshots.get(3)[0] + assertEquals(0L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterInsertSnapshot} + """)) + assertEquals(1L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterFirstDeleteSnapshot} + """)) + assertEquals(2L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterSecondDeleteSnapshot} + """)) + assertEquals([[1, "a"], [2, "b"], [3, "c"]], + sql("""select * from pd_schema_time_travel for version as of ${afterInsertSnapshot} order by id""")) + assertEquals([[1, "a"], [3, "c"]], + sql("""select * from pd_schema_time_travel for version as of ${afterFirstDeleteSnapshot} order by id""")) + assertEquals([[1, "a", null], [3, "c", null], [5, "e", "new"]], + sql("""select * from pd_schema_time_travel for version as of ${afterSecondDeleteSnapshot} order by id""")) + + assertPositionDeletesSchema("pd_no_deletes", false) + assertSparkDorisPositionDeletes("pd_no_deletes", commonCompareColumns) + assertEquals(0L, countRows("""select count(*) from pd_no_deletes\$position_deletes""")) + + def assertPositionDeletesScannerPath = { boolean enableFileScannerV2 -> + sql """set enable_file_scanner_v2 = ${enableFileScannerV2}""" + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where pos >= 0""")) + assertEquals(partitionedCount, countRows( + """select count(*) from pd_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(partitionRenameCount, countRows( + """select count(*) from pd_partition_rename\$position_deletes + where struct_element(`partition`, 'p2') is not null""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertSparkDorisPositionDeletes("pd_orc_unpartitioned", commonCompareColumns) + assertSparkDorisPositionDeletes("pd_v3_unpartitioned", v3CompareColumns) + } + assertPositionDeletesScannerPath(false) + assertPositionDeletesScannerPath(true) + sql """set enable_file_scanner_v2 = true""" + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.pd_doris_spark_interop; + CREATE TABLE demo.${dbName}.pd_doris_spark_interop ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_doris_spark_interop + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'spark_a', '2026-07-03'), + (2, 'spark_b', '2026-07-03') AS t(id, name, dt); + """ + + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"] + ]) + + sql """insert into pd_doris_spark_interop values (3, 'doris_c', '2026-07-04')""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"], + [3, "doris_c", "2026-07-04"] + ]) + + spark_iceberg """ + INSERT INTO demo.${dbName}.pd_doris_spark_interop + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (4, 'spark_d', '2026-07-04') AS t(id, name, dt) + """ + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"], + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + sql """delete from pd_doris_spark_interop where id = 2""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + spark_iceberg """DELETE FROM demo.${dbName}.pd_doris_spark_interop WHERE id = 1""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + assertPositionDeletesSchema("pd_doris_spark_interop", true) + long interopPositionDeleteCount = countRows( + """select count(*) from pd_doris_spark_interop\$position_deletes""") + assertEquals(2L, interopPositionDeleteCount) + assertEquals(interopPositionDeleteCount, countRows( + """select count(*) from pd_doris_spark_interop\$position_deletes + where `partition` is not null and delete_file_path is not null""")) + assertSparkDorisPositionDeletes("pd_doris_spark_interop", commonCompareColumns) +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy index 3e541af65d7818..cb01ba95bb72ac 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy @@ -262,7 +262,7 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock systableName) } - def test_table_systables = { table -> + def test_table_systables = { table, boolean partitioned -> test_systable_entries(table, "entries") test_systable_entries(table, "all_entries") test_systable_files(table, "files") @@ -278,17 +278,21 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock test_systable_manifests(table, "manifests") test_systable_manifests(table, "all_manifests") test_systable_partitions(table) - // TODO: these table will be supportted in future - // test_systable_position_deletes(table) - test { - sql """select * from ${table}\$position_deletes""" - exception "SysTable position_deletes is not supported yet" + List> positionDeleteDesc = sql """desc ${table}\$position_deletes""" + List positionDeleteColumns = positionDeleteDesc.collect { it[0].toString() } + List expectedPositionDeleteColumns = ["file_path", "pos", "row"] + if (partitioned) { + expectedPositionDeleteColumns.add("partition") } + expectedPositionDeleteColumns.addAll(["spec_id", "delete_file_path"]) + assertEquals(expectedPositionDeleteColumns, positionDeleteColumns) + List> positionDeleteCount = sql """select count(*) from ${table}\$position_deletes""" + assertEquals(1, positionDeleteCount.size()) } - test_table_systables("test_iceberg_systable_unpartitioned") - test_table_systables("test_iceberg_systable_partitioned") + test_table_systables("test_iceberg_systable_unpartitioned", false) + test_table_systables("test_iceberg_systable_partitioned", true) sql """drop table if exists test_iceberg_systable_tbl1;""" sql """create table test_iceberg_systable_tbl1 (id int);""" @@ -319,10 +323,17 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock """ exception "denied" } + test { + sql """ + select file_path, pos from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$position_deletes + """ + exception "denied" + } } sql """grant select_priv on ${catalog_name}.${db_name}.test_iceberg_systable_tbl1 to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { sql """select committed_at, snapshot_id, parent_id, operation from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$snapshots""" + sql """select file_path, pos from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$position_deletes""" } try_sql("DROP USER ${user}") From bad58beafa5cb07a89da0d5ed4bf58c89059e8a4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 23:04:50 +0800 Subject: [PATCH 08/24] [fix](regression) Wait past unsafe time boundaries (#65742) The regression framework truncated the remaining time before an hour or day boundary to whole seconds. Sleeping for that truncated value could resume in the final fractional second before the boundary, allowing a boundary-sensitive case to cross it and fail intermittently. This change computes the wait in milliseconds and sleeps one additional millisecond so execution resumes strictly after the boundary. It also adds focused coverage for fractional, exact, and safe boundary windows and aligns the JUnit API with the engine already provided by Groovy. --- regression-test/framework/pom.xml | 2 +- .../doris/regression/suite/Suite.groovy | 34 +++++------ .../suite/SuiteBoundaryWaitTest.groovy | 56 +++++++++++++++++++ 3 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index 9efd3b6fe97cd4..c5b81b39ce38db 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -210,7 +210,7 @@ under the License. org.junit.jupiter junit-jupiter-api - 5.8.2 + 5.10.2 mysql diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 0075e501545a3d..70e5c6e11a4d7d 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -3655,35 +3655,35 @@ class Suite implements GroovyInterceptable { throw new IllegalArgumentException("invalid caseElapseSeconds, ${caseElapseSeconds}") } - long sleepSeconds = 0 - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now() + LocalDateTime boundary switch (caseSpanConstraint) { case "NOT_CROSS_HOUR_BOUNDARY": - LocalDateTime nextHour = now.withMinute(0).withSecond(0).withNano(0).plusHours(1); - long secondsToNextHour = ChronoUnit.SECONDS.between(now, nextHour) - - if (secondsToNextHour < caseElapseSeconds) { - sleepSeconds = secondsToNextHour - } + boundary = now.withMinute(0).withSecond(0).withNano(0).plusHours(1) break case "NOT_CROSS_DAY_BOUNDARY": - LocalDateTime startOfNextDay = now.toLocalDate().plusDays(1).atStartOfDay(); - long secondsToNextDay = ChronoUnit.SECONDS.between(now, startOfNextDay) - - if (secondsToNextDay < caseElapseSeconds) { - sleepSeconds = secondsToNextDay - } + boundary = now.toLocalDate().plusDays(1).atStartOfDay() break default: throw new IllegalArgumentException("invalid caseSpanConstraint:${caseSpanConstraint}") } - if (sleepSeconds > 0) { - logger.info("test sleeps ${sleepSeconds} to satisfy ${caseSpanConstraint}") - Thread.sleep(sleepSeconds * 1000) + long sleepMillis = calculateBoundarySleepMillis(now, boundary, caseElapseSeconds) + if (sleepMillis > 0) { + logger.info("test sleeps ${sleepMillis} ms to satisfy ${caseSpanConstraint}") + Thread.sleep(sleepMillis) + } + } + + static long calculateBoundarySleepMillis(LocalDateTime now, LocalDateTime boundary, int caseElapseSeconds) { + long millisToBoundary = ChronoUnit.MILLIS.between(now, boundary) + if (millisToBoundary <= TimeUnit.SECONDS.toMillis(caseElapseSeconds)) { + // Cross the boundary by a full millisecond; truncated fractional milliseconds must not resume early. + return millisToBoundary + 1 } + return 0 } void retryUntilHasSqlCache(String sql) { diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy new file mode 100644 index 00000000000000..4b8693e7678d6a --- /dev/null +++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import org.junit.jupiter.api.Test + +import java.time.LocalDateTime +import java.util.concurrent.TimeUnit + +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertTrue + +class SuiteBoundaryWaitTest { + @Test + void waitCrossesBoundaryWhenRemainingTimeHasFractionalSecond() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:40.285999999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void waitCrossesBoundaryWhenExpectedDurationEndsExactlyAtBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:15") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void noWaitWhenExpectedDurationFinishesBeforeBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:14.999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + assertEquals(0, Suite.calculateBoundarySleepMillis(now, nextHour, 45)) + } +} From 59b673fdee5785a9e9c910a242cecf6b56479b6d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 23:06:32 +0800 Subject: [PATCH 09/24] [fix](fe) Support ClickHouse JDBC V2 metadata (#65709) ClickHouse JDBC V2 exposes databases as schemas and reports distributed tables as `REMOTE TABLE`. Doris inferred the database metadata mode from the legacy `databaseterm` URL parameter and filtered the vendor table type, so V2 catalogs could not discover databases or distributed tables. This change uses JDBC metadata capabilities to select catalog/schema mode and includes remote tables in discovery in both JDBC client implementations. --- .../clickhouse/init/03-create-table.sql | 10 ++++- .../jdbc/client/JdbcClickHouseClient.java | 23 ++++------ .../jdbc/client/JdbcClickHouseClientTest.java | 25 +++++++++++ .../jdbc/test_clickhouse_jdbc_v2.out | 9 ++++ .../jdbc/test_clickhouse_jdbc_v2.groovy | 43 +++++++++++++++++++ 5 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out create mode 100644 regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql index 039ddd1d3690fb..d45b878d1ed29d 100644 --- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql @@ -342,6 +342,14 @@ CREATE TABLE doris_test.extreme_test ) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE doris_test.distributed_type AS doris_test.type +ENGINE = Distributed(default, doris_test, type, rand()); + +CREATE MATERIALIZED VIEW doris_test.materialized_view_type +ENGINE = MergeTree +ORDER BY k1 +AS SELECT * FROM doris_test.type; + CREATE TABLE doris_test.extreme_test_multi_block ( id UInt64, @@ -392,4 +400,4 @@ CREATE TABLE doris_test.extreme_test_multi_block ipv6_col IPv6, ipv6_nullable Nullable(IPv6) ) ENGINE = MergeTree() -ORDER BY id; \ No newline at end of file +ORDER BY id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java index 3837b7bbc7979f..fbe2889d8cebaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java @@ -39,12 +39,9 @@ public class JdbcClickHouseClient extends JdbcClient { protected JdbcClickHouseClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); try (Connection conn = getConnection()) { - String jdbcUrl = conn.getMetaData().getURL(); - if (!isNewClickHouseDriver(getJdbcDriverVersion())) { - this.databaseTermIsCatalog = false; - } else { - this.databaseTermIsCatalog = "catalog".equalsIgnoreCase(getDatabaseTermFromUrl(jdbcUrl)); - } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + this.databaseTermIsCatalog = isDatabaseTermCatalog( + databaseMetaData, databaseMetaData.getDriverVersion()); } catch (SQLException e) { throw new JdbcClientException("Failed to initialize JdbcClickHouseClient: %s", e.getMessage()); } @@ -128,7 +125,8 @@ protected String getRemoteDatabaseName(ResultSet resultSet) throws SQLException @Override protected String[] getTableTypes() { - return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"}; + // ClickHouse JDBC V2 filters engines by these vendor-specific table type names. + return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}; } @Override @@ -236,14 +234,9 @@ private static boolean isNewClickHouseDriver(String driverVersion) { } } - /** - * Extract databaseterm parameters from the jdbc url. - */ - private String getDatabaseTermFromUrl(String jdbcUrl) { - if (jdbcUrl != null && jdbcUrl.toLowerCase().contains("databaseterm=schema")) { - return "schema"; - } - return "catalog"; + static boolean isDatabaseTermCatalog(DatabaseMetaData databaseMetaData, String driverVersion) + throws SQLException { + return isNewClickHouseDriver(driverVersion) && databaseMetaData.supportsCatalogsInDataManipulation(); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java index 99e4aa62dd574d..a76319d108f7cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java @@ -19,11 +19,36 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Answers; +import org.mockito.Mockito; import java.lang.reflect.Method; +import java.sql.DatabaseMetaData; public class JdbcClickHouseClientTest { + @Test + public void testDatabaseTermFollowsDriverMetadata() throws Exception { + DatabaseMetaData databaseMetaData = Mockito.mock(DatabaseMetaData.class); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(false); + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.9.8")); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(true); + Assert.assertTrue(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.7.1")); + + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.4.2")); + } + + @Test + public void testClickHouseSpecificTableTypesAreVisible() { + JdbcClickHouseClient client = Mockito.mock(JdbcClickHouseClient.class, Answers.CALLS_REAL_METHODS); + + Assert.assertArrayEquals( + new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}, + client.getTableTypes()); + } + @Test public void testIsNewClickHouseDriver() { try { diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out new file mode 100644 index 00000000000000..0ad208e4be8889 --- /dev/null +++ b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !clickhouse_v2_schema -- +2 + +-- !clickhouse_v2_distributed -- +2 + +-- !clickhouse_v2_materialized_view -- +2 diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy new file mode 100644 index 00000000000000..4d986201fbeea4 --- /dev/null +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_clickhouse_jdbc_v2", "p0,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String clickhousePort = context.config.otherConfigs.get("clickhouse_22_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + // Keep large drivers in the regression bucket so isolated workers do not require public egress. + String driverUrl = "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.9.8-all.jar" + + sql """ drop catalog if exists clickhouse_v2_schema """ + sql """ create catalog clickhouse_v2_schema properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhousePort}/doris_test?jdbc_schema_term=schema", + "driver_url" = "${driverUrl}", + "driver_class" = "com.clickhouse.jdbc.Driver" + );""" + + order_qt_clickhouse_v2_schema """ select count(*) from clickhouse_v2_schema.doris_test.type """ + order_qt_clickhouse_v2_distributed """ select count(*) from clickhouse_v2_schema.doris_test.distributed_type """ + order_qt_clickhouse_v2_materialized_view """ select count(*) from clickhouse_v2_schema.doris_test.materialized_view_type """ + sql """ drop catalog clickhouse_v2_schema """ + } +} From 388c55deb21a51edc40d14b144ca5bb90c0199a0 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 10:06:49 +0800 Subject: [PATCH 10/24] [fix](scanner)(nereids) Harden FileScannerV2 and fix external COUNT pushdown semantics (#65548) Issue Number: None Related PR: None This PR fixes two correctness areas in external file scans. External `COUNT(*)` retained an arbitrary scan slot after column pruning. BE then inferred aggregate semantics from that post-pruning scan shape and could treat the retained nullable placeholder as the argument of `COUNT(column)`. This produced wrong results, including 9015 instead of 10000 rows for special ORC data and 116 instead of 219 rows for Hive basic types. The fix transports semantic COUNT arguments from Nereids through `TPlanNode.push_down_count_slot_ids` to both file-scanner implementations and their table/hybrid readers. The thrift field has an explicit compatibility contract: - absent: an old FE plan with unknown COUNT semantics; use the conservative normal-scan path; - present and empty: explicit `COUNT(*)` / row-count semantics; - present and non-empty: explicit `COUNT(column)` arguments, which may use metadata only when every mapping is proven safe. Nereids no longer applies storage COUNT pushdown to cast arguments whose null/error behavior would be lost. FE split planning also restricts Iceberg/Paimon metadata-count split reduction and Hive/TVF no-split shortcuts to explicit row-count semantics, so a `COUNT(column)` fallback retains all data splits and normal scan parallelism. BE V1, V2, Hudi, Paimon, and table-reader paths preserve the same argument state, reject unsafe metadata shortcuts, and keep adaptive batching enabled for real-row fallbacks. FileScannerV2 treated valid empty splits as failures, could count stopped EOF or malformed Native input as an empty file, localized evolved nested predicates without fully preserving type/nullability contracts, and rejected whole Parquet schemas when only unprojected leaves used unsupported logical types. Parquet page-cache range metadata also lost warm non-exact reuse while lacking a bound. The fix distinguishes stopped reads, valid empty input, and malformed/truncated input; keeps unsafe nested predicates above `TableReader`; validates only semantically required Parquet projections while preserving strict checks for real predicate and `COUNT(column)` inputs; uses safe row-position/default carriers for placeholder-only paths; and shares bounded per-file page-range indexes without adding a process-wide lock to the `ReadAt` hot path. Fix wrong external `COUNT(*)` / `COUNT(column)` results and unsafe metadata pushdown across Nereids, thrift, FE split planning, and BE file readers. Also harden FileScannerV2 handling of empty or interrupted input, evolved nested predicates, unsupported unprojected Parquet logical types, COUNT placeholders, hybrid readers, and bounded cross-reader page-cache range reuse. - Test: Regression test / Unit Test - BE ASAN unit coverage for V1/V2 COUNT semantics, table-level metadata gates, schema-evolution fallbacks, COUNT placeholders, adaptive batching, and Hudi/Paimon/Iceberg hybrid paths - FE unit coverage for Nereids COUNT argument capture and Hive/Iceberg/Paimon/TVF split-planning behavior - `ColumnMapper*.*` BE ASAN suite (111 tests passed) - Existing external regression cases `test_special_orc_formats` and `test_hive_basic_type` reproduce the fixed COUNT(*) wrong results - `test_file_scanner_v2_review_fixes` regression suite covers empty input, evolved nested types, unsupported Parquet placeholders, and COUNT fallback behavior - Behavior changed: Yes (COUNT semantics are transported explicitly; unsafe or ambiguous metadata COUNT plans fall back to normal scanning; valid empty splits are skipped; stopped or malformed input is not counted as empty; unsupported unprojected Parquet leaves no longer fail unrelated scans) - Does this need documentation: No --- be/src/exec/operator/scan_operator.cpp | 9 + be/src/exec/operator/scan_operator.h | 13 + be/src/exec/scan/file_scanner.h | 34 +- be/src/exec/scan/file_scanner_v2.cpp | 78 ++++- be/src/exec/scan/file_scanner_v2.h | 10 + be/src/format_v2/column_mapper.cpp | 282 +++++++++++++--- be/src/format_v2/file_reader.cpp | 9 +- be/src/format_v2/file_reader.h | 12 + be/src/format_v2/native/native_reader.cpp | 28 +- .../parquet/parquet_column_schema.cpp | 13 +- .../parquet/parquet_file_context.cpp | 154 +++++---- .../format_v2/parquet/parquet_file_context.h | 36 ++ be/src/format_v2/parquet/parquet_reader.cpp | 89 ++++- be/src/format_v2/parquet/parquet_scan.cpp | 45 ++- be/src/format_v2/parquet/parquet_type.cpp | 3 +- be/src/format_v2/parquet/parquet_type.h | 3 + be/src/format_v2/table/hudi_reader.cpp | 6 + be/src/format_v2/table/hudi_reader.h | 1 + be/src/format_v2/table/iceberg_reader.cpp | 15 +- be/src/format_v2/table/paimon_reader.cpp | 6 + be/src/format_v2/table/paimon_reader.h | 1 + be/src/format_v2/table_reader.cpp | 10 +- be/src/format_v2/table_reader.h | 102 ++++-- be/test/exec/scan/file_scanner_v2_test.cpp | 33 ++ be/test/format_v2/column_mapper_test.cpp | 315 ++++++++++++++++++ .../format_v2/native/native_reader_test.cpp | 37 +- .../parquet/parquet_page_cache_range_test.cpp | 39 +++ .../parquet/parquet_reader_control_test.cpp | 14 + .../format_v2/parquet/parquet_scan_test.cpp | 128 +++++++ .../format_v2/parquet/parquet_schema_test.cpp | 26 +- be/test/format_v2/table/hudi_reader_test.cpp | 99 ++++++ .../format_v2/table/iceberg_reader_test.cpp | 76 +++++ .../format_v2/table/paimon_reader_test.cpp | 53 +++ be/test/format_v2/table_reader_test.cpp | 297 ++++++++++++++++- .../apache/doris/datasource/FileScanNode.java | 17 + .../datasource/hive/source/HiveScanNode.java | 5 +- .../iceberg/source/IcebergScanNode.java | 4 +- .../paimon/source/PaimonScanNode.java | 5 +- .../datasource/tvf/source/TVFScanNode.java | 5 +- .../translator/PhysicalPlanTranslator.java | 10 + .../translator/PlanTranslatorContext.java | 10 + .../implementation/AggregateStrategies.java | 30 +- .../PhysicalStorageLayerAggregate.java | 32 +- .../org/apache/doris/planner/PlanNode.java | 7 + .../iceberg/source/IcebergScanNodeTest.java | 51 +++ .../paimon/source/PaimonScanNodeTest.java | 50 +++ .../tvf/source/TVFScanNodeTest.java | 55 +++ .../PhysicalStorageLayerAggregateTest.java | 25 +- gensrc/thrift/PlanNodes.thrift | 4 + .../tvf/file_scanner_v2_empty.csv | 0 .../file_scanner_v2_struct_0_bigint.parquet | Bin 0 -> 965 bytes .../tvf/file_scanner_v2_struct_1_int.parquet | Bin 0 -> 923 bytes .../file_scanner_v2_unsupported_time.parquet | Bin 0 -> 673 bytes .../tvf/test_file_scanner_v2_review_fixes.out | 17 + .../test_file_scanner_v2_review_fixes.groovy | 113 +++++++ 55 files changed, 2321 insertions(+), 195 deletions(-) create mode 100644 regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv create mode 100644 regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet create mode 100644 regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet create mode 100644 regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet create mode 100644 regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out create mode 100644 regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index ae2b84cc98e72e..f945fa0a488810 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -1011,6 +1011,12 @@ TPushAggOp::type ScanLocalState::get_push_down_agg_type() { return _parent->cast()._push_down_agg_type; } +template +const std::optional>& ScanLocalState::get_push_down_count_slot_ids() + const { + return _parent->cast()._push_down_count_slot_ids; +} + template int64_t ScanLocalState::limit_per_scanner() { return _parent->cast()._limit_per_scanner; @@ -1164,6 +1170,9 @@ Status ScanOperatorX::init(const TPlanNode& tnode, RuntimeState* } else { _push_down_agg_type = TPushAggOp::type::NONE; } + if (tnode.__isset.push_down_count_slot_ids) { + _push_down_count_slot_ids = tnode.push_down_count_slot_ids; + } if (tnode.__isset.topn_filter_source_node_ids) { _topn_filter_source_node_ids = tnode.topn_filter_source_node_ids; diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index 931917482736a6..fb7d5ec992c793 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -73,6 +74,7 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual void set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) = 0; virtual TPushAggOp::type get_push_down_agg_type() = 0; + virtual const std::optional>& get_push_down_count_slot_ids() const = 0; // If scan operator is serial operator(like topn), its real parallelism is 1. // Otherwise, its real parallelism is query_parallel_instance_num. @@ -231,6 +233,7 @@ class ScanLocalState : public ScanLocalStateBase { const std::vector& scan_ranges) override {} TPushAggOp::type get_push_down_agg_type() override; + const std::optional>& get_push_down_count_slot_ids() const override; std::vector execution_dependencies() override { if (_filter_dependencies.empty()) { @@ -409,6 +412,16 @@ class ScanOperatorX : public OperatorX { TPushAggOp::type _push_down_agg_type; + // Semantic arguments of a pushed-down COUNT. This is deliberately optional because absence + // and an empty list have different meanings during a BE-first rolling upgrade: + // + // - nullopt: an old FE did not send the field, so the new BE must use the normal scan; + // - empty: the new FE explicitly planned COUNT(*)/COUNT(1); + // - non-empty: the new FE explicitly planned COUNT(col). + // + // Treating nullopt as empty would silently reinterpret an old plan as COUNT(*). + std::optional> _push_down_count_slot_ids; + // Record the value of the aggregate function 'count' from doris's be int64_t _push_down_count = -1; const int _parallel_tasks = 0; diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index b4c0c172100fa4..35f29f2b54c471 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -77,6 +78,10 @@ class FileScanner : public Scanner { const VExprContextSPtrs& TEST_runtime_filter_partition_prune_ctxs() const { return _runtime_filter_partition_prune_ctxs; } + static TPushAggOp::type TEST_effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + return _effective_push_down_agg_type(agg_type, count_slot_ids); + } #endif FileScanner(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -305,11 +310,6 @@ class FileScanner : public Scanner { _counter.num_rows_filtered = 0; } - TPushAggOp::type _get_push_down_agg_type() const { - return _local_state == nullptr ? TPushAggOp::type::NONE - : _local_state->get_push_down_agg_type(); - } - void _reset_adaptive_batch_size_state(); void _init_adaptive_batch_size_state(TFileFormatType::type format_type); bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; @@ -318,6 +318,30 @@ class FileScanner : public Scanner { void _update_adaptive_batch_size_before_truncate(const Block& block); void _update_adaptive_batch_size_after_truncate(const Block& block); + static TPushAggOp::type _effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + if (agg_type != TPushAggOp::type::COUNT) { + return agg_type; + } + // V1's CountReader receives only the file's total row count and emits that many synthetic + // rows. This is exact for COUNT(*)/COUNT(1), but it has no column metadata for NULL or CAST + // semantics. For example, a 10,000-row file with 9,015 non-null values must return 10,000 + // for COUNT(*) and 9,015 for COUNT(nullable_col); CountReader can produce only the former. + // Therefore a non-empty argument list must use the normal reader. nullopt is an old FE plan + // that predates the argument field; treating it as empty would silently reinterpret unknown + // semantics as COUNT(*). + return count_slot_ids.has_value() && count_slot_ids->empty() ? TPushAggOp::type::COUNT + : TPushAggOp::type::NONE; + } + + TPushAggOp::type _get_push_down_agg_type() const { + if (_local_state == nullptr) { + return TPushAggOp::type::NONE; + } + return _effective_push_down_agg_type(_local_state->get_push_down_agg_type(), + _local_state->get_push_down_count_slot_ids()); + } + // enable the file meta cache only when // 1. max_external_file_meta_cache_num is > 0 // 2. the file number is less than 1/3 of cache's capacibility diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 5b593ccd178604..daf179e36f1fe7 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -266,6 +266,10 @@ void FileScannerV2::TEST_report_file_cache_profile( bool FileScannerV2::TEST_should_skip_not_found(const Status& status, bool ignore_not_found) { return _should_skip_not_found(status, ignore_not_found); } + +bool FileScannerV2::TEST_should_skip_empty(const Status& status, bool stopped) { + return _should_skip_empty(status, stopped); +} #endif bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -316,6 +320,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc RETURN_IF_ERROR(Scanner::init(state, conjuncts)); _get_block_timer = ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); + _empty_file_counter = + ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT, 1); _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "NotFoundFileNum", TUnit::UNIT, 1); _file_counter = @@ -388,6 +394,20 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e *eof = false; continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // END_OF_FILE here means the reader discovered a valid split with no data while + // opening or probing it, not that the Scanner has exhausted all splits. Examples + // are a zero-byte CSV with an explicit schema and a Doris Native file containing + // only its 12-byte header. Treat it like V1's empty-file path: finish this range, + // discard partial reader state, and let the loop fetch the next split. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + _has_prepared_split = false; + block->clear_column_data(cast_set(_projected_columns.size())); + *eof = false; + continue; + } RETURN_IF_ERROR(status); } if (*eof) { @@ -417,9 +437,12 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { const auto format_type = get_range_format_type(*_params, _current_range); _init_adaptive_batch_size_state(format_type); - if (_should_run_adaptive_batch_size()) { - // JNI readers open eagerly in prepare_split(). Seed the probe size first so readers - // such as Paimon also use it for their first physical read batch. + if (_block_size_predictor != nullptr) { + // JNI readers open eagerly in prepare_split(). Always seed the probe before preparing + // the next split: its metadata-COUNT decision is not available yet, and the state + // exposed by TableReader can still describe the preceding split. Metadata shortcuts + // ignore this batch size, while row-scan fallbacks need it for their first physical + // read batch. _table_reader->set_batch_size(_predict_reader_batch_rows()); } std::map partition_values; @@ -432,6 +455,16 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { _state->update_num_finished_scan_range(1); continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // Schema discovery can reach EOF before a split becomes prepared. A header-only Native + // file follows this path, while a reader that discovers emptiness on its first + // get_block() follows the symmetric branch in _get_block_impl(). Both paths must + // advance exactly one scan range and preserve later files in the same scan. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + continue; + } RETURN_IF_ERROR(status); if (_table_reader->current_split_pruned()) { _state->update_num_finished_scan_range(1); @@ -452,6 +485,21 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + std::optional> push_down_count_columns; + const auto& push_down_count_slot_ids = _local_state->get_push_down_count_slot_ids(); + if (push_down_count_slot_ids.has_value()) { + push_down_count_columns.emplace(); + push_down_count_columns->reserve(push_down_count_slot_ids->size()); + for (const auto slot_id : *push_down_count_slot_ids) { + const auto global_index_it = _slot_id_to_global_index.find(slot_id); + if (global_index_it == _slot_id_to_global_index.end()) { + return Status::InternalError( + "Pushed-down COUNT argument is not a projected file scan slot, slot_id={}", + slot_id); + } + push_down_count_columns->push_back(global_index_it->second); + } + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), @@ -462,6 +510,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .scanner_profile = _local_state->scanner_profile(), .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), + .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), })); return Status::OK(); @@ -534,6 +583,14 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not return ignore_not_found && status.is(); } +bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) { + // Several readers use END_OF_FILE both for a valid zero-row split and for an interrupted IO. + // For example, DeletionVectorReader returns END_OF_FILE("stop read.") after try_stop() marks + // the shared IOContext. That status must unwind the stopped scanner; counting it as an empty + // file would incorrectly finish the scan range and increment EmptyFileNum. + return !stopped && status.is(); +} + bool FileScannerV2::_should_enable_file_meta_cache() const { return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; @@ -800,10 +857,17 @@ bool FileScannerV2::_should_enable_adaptive_batch_size(TFileFormatType::type for } bool FileScannerV2::_should_run_adaptive_batch_size() const { - // COUNT pushdown emits synthetic rows from file metadata and does not materialize file columns, - // so there is no useful row-width sample to learn from. - return _block_size_predictor != nullptr && - _local_state->get_push_down_agg_type() != TPushAggOp::type::COUNT; + DORIS_CHECK(_table_reader != nullptr); + return _should_run_adaptive_batch_size(_block_size_predictor != nullptr, + _table_reader->current_split_uses_metadata_count()); +} + +bool FileScannerV2::_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + // Metadata COUNT emits synthetic rows and has no physical row width to learn from. A raw COUNT + // opcode is not sufficient here: unsupported argument counts, mappings, filters, or deletes + // make TableReader fall back to materializing normal rows, which still need adaptive batching. + return predictor_initialized && !current_split_uses_metadata_count; } size_t FileScannerV2::_predict_reader_batch_rows() { diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 2bddc5d5e69e6c..0cd03030b56851 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -87,6 +87,12 @@ class FileScannerV2 final : public Scanner { static void TEST_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); + static bool TEST_should_skip_empty(const Status& status, bool stopped); + static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + return _should_run_adaptive_batch_size(predictor_initialized, + current_split_uses_metadata_count); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -121,6 +127,7 @@ class FileScannerV2 final : public Scanner { Status _prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); + static bool _should_skip_empty(const Status& status, bool stopped); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -138,6 +145,8 @@ class FileScannerV2 final : public Scanner { void _init_adaptive_batch_size_state(TFileFormatType::type format_type); bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; bool _should_run_adaptive_batch_size() const; + static bool _should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count); size_t _predict_reader_batch_rows(); void _update_adaptive_batch_size(const Block& block); static RealtimeCounterDeltas _collect_realtime_counter_deltas( @@ -181,6 +190,7 @@ class FileScannerV2 final : public Scanner { ShardedKVCache* _kv_cache = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; + RuntimeProfile::Counter* _empty_file_counter = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 61f35023963eca..4c23bc9f83380b 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -811,6 +811,19 @@ static bool collect_struct_element_chain(const VExprSPtr& expr, std::vector, table + // STRUCT, rows [NULL, 20], and `s.a > 10`, filtering in the file domain would drop + // NULL first and hide the table-contract violation. The reverse direction is safe: a required + // file value can always be wrapped as a nullable table value after filtering. + return !file_type->is_nullable() || table_type->is_nullable(); +} + static bool rewrite_struct_element_path_to_file_expr( const VExprSPtr& expr, const std::vector& mappings, const std::map& global_to_file_slot, @@ -837,6 +850,22 @@ static bool rewrite_struct_element_path_to_file_expr( return false; } + // Check every value-producing level, including the root struct. A nullable parent also makes + // a child access nullable even when the child type itself is required, so checking only the + // final leaf is insufficient. If any file level is more nullable than its table counterpart, + // keep the complete predicate above TableReader so schema validation observes all NULLs before + // row filtering. + if (!can_filter_before_table_nullability_alignment(rewrite_it->second.file_type, + rewrite_it->second.table_type)) { + return false; + } + for (size_t idx = 0; idx < struct_element_chain.size(); ++idx) { + if (!can_filter_before_table_nullability_alignment( + resolved.file_child_types[idx], struct_element_chain[idx]->data_type())) { + return false; + } + } + // File-local conjuncts are prepared against the file-reader Block, so both the root slot and // every struct selector must be expressed in file schema terms. For a renamed Iceberg field, // keeping the table selector would prepare `element_at(file_struct, 'renamed')` and @@ -865,6 +894,209 @@ static bool rewrite_struct_element_path_to_file_expr( return true; } +static VExprSPtr cast_file_expr_to_table_type(const VExprSPtr& file_expr, + const DataTypePtr& table_type, + RewriteContext* rewrite_context) { + DORIS_CHECK(file_expr != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(rewrite_context != nullptr); + auto cast_expr = Cast::create_shared(table_type); + cast_expr->add_child(file_expr); + rewrite_context->add_created_expr(cast_expr); + return cast_expr; +} + +// Prefer comparing in the physical file leaf type when a table predicate uses a promoted struct +// child. For example, with table STRUCT, old-file STRUCT, and `s.a = 10`, the +// localized predicate should be `file_s.a::INT = 10::INT`, not +// `CAST(file_s.a::INT AS BIGINT) = 10::BIGINT`. Converting one literal avoids a cast for every row. +// +// This rewrite is valid only when every possible file value survives file-to-table conversion and +// the particular literal survives a table-to-file-to-table round trip. A value such as BIGINT +// 2147483648 cannot be represented by an INT file leaf, so that case deliberately falls back to +// `CAST(file_s.a AS BIGINT) = 2147483648`, which preserves the original table-level semantics. +static bool rewrite_binary_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (!is_binary_comparison_predicate(expr)) { + return false; + } + auto children = expr->children(); + int struct_child_idx = -1; + int literal_child_idx = -1; + if (is_struct_element_expr(children[0])) { + struct_child_idx = 0; + literal_child_idx = 1; + } else if (is_struct_element_expr(children[1])) { + struct_child_idx = 1; + literal_child_idx = 0; + } else { + return false; + } + + const auto table_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type); + if (table_literal == nullptr || + !rewrite_struct_element_path_to_file_expr(children[struct_child_idx], filter_mappings, + global_to_file_slot, rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal != nullptr) { + children[literal_child_idx] = std::move(file_literal); + } else { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + // A narrowing or otherwise lossy cast can fail or produce NULL while TableReader + // materializes the table schema. Evaluating it here could filter the offending row + // before that validation, so keep the complete predicate above TableReader. + *can_localize = false; + return true; + } + children[struct_child_idx] = cast_file_expr_to_table_type(children[struct_child_idx], + table_leaf_type, rewrite_context); + children[literal_child_idx] = original_table_literal(table_literal, rewrite_context); + } + expr->set_children(std::move(children)); + return true; +} + +// IN must use one comparison type for its probe and every candidate. Rewrite the complete literal +// set only when all values are exactly representable in the file leaf type; one unsafe value makes +// the whole predicate fall back to a table-type cast. For example, an INT file leaf can evaluate +// `BIGINT IN (10, 20)` as `INT IN (10, 20)`, but `BIGINT IN (10, 2147483648)` must stay BIGINT. +static bool rewrite_in_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (expr->node_type() != TExprNodeType::IN_PRED || expr->get_num_children() < 2 || + !is_struct_element_expr(expr->children()[0])) { + return false; + } + auto children = expr->children(); + const auto table_leaf_type = children[0]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + VExprSPtrs table_literals; + table_literals.reserve(children.size() - 1); + for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { + auto table_literal = unwrap_literal_for_file_cast(children[child_idx], table_leaf_type); + if (table_literal == nullptr) { + return false; + } + table_literals.push_back(std::move(table_literal)); + } + if (!rewrite_struct_element_path_to_file_expr(children[0], filter_mappings, global_to_file_slot, + rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[0]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + VExprSPtrs file_literals; + file_literals.reserve(table_literals.size()); + for (const auto& table_literal : table_literals) { + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal == nullptr) { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + *can_localize = false; + return true; + } + children[0] = + cast_file_expr_to_table_type(children[0], table_leaf_type, rewrite_context); + for (size_t literal_idx = 0; literal_idx < table_literals.size(); ++literal_idx) { + children[literal_idx + 1] = + original_table_literal(table_literals[literal_idx], rewrite_context); + } + expr->set_children(std::move(children)); + return true; + } + file_literals.push_back(std::move(file_literal)); + } + + for (size_t literal_idx = 0; literal_idx < file_literals.size(); ++literal_idx) { + children[literal_idx + 1] = std::move(file_literals[literal_idx]); + } + expr->set_children(std::move(children)); + return true; +} + +static VExprSPtr rewrite_struct_or_slot_expr_to_file_expr( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + const std::vector& filter_mappings, RewriteContext* rewrite_context, + bool* can_localize) { + if (is_struct_element_expr(expr)) { + const auto table_leaf_type = expr->data_type(); + if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, + rewrite_context)) { + // The scanner still evaluates the original table-level conjunct after TableReader + // finalizes the output block. Skipping an unlocalizable file conjunct is therefore + // safer than preparing a partially rewritten expression against the wrong struct + // layout. In particular, do not generate file-local conjuncts for computed complex + // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct + // slot-rooted struct chains are supported here. + *can_localize = false; + return expr; + } + DORIS_CHECK(table_leaf_type != nullptr); + DORIS_CHECK(expr->data_type() != nullptr); + if (!expr->data_type()->equals(*table_leaf_type)) { + if (!is_lossless_file_to_table_numeric_cast(expr->data_type(), table_leaf_type)) { + *can_localize = false; + return expr; + } + // Path localization changes the leaf to the physical file type. For example, after an + // Iceberg evolution from STRUCT to STRUCT, the localized old-file + // predicate is initially `element_at(file_col, 'a')::INT = 10::BIGINT`. Cast only the + // leaf back to BIGINT so the comparison has matching operands without forcing a cast + // of the entire evolved struct (whose children may also have been added or reordered). + return cast_file_expr_to_table_type(expr, table_leaf_type, rewrite_context); + } + return expr; + } + + DORIS_CHECK(expr->is_slot_ref()); + const auto* slot_ref = assert_cast(expr.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it == global_to_file_slot.end()) { + return expr; + } + const auto& rewrite_info = rewrite_it->second; + auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); + if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { + return file_slot; + } + if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { + // Generic file-local expressions cannot safely cast an evolved complex file slot back to + // the table type. For example, ARRAY_CONTAINS(MAP_KEYS(m), 'person5') only reads map keys, + // but CAST(file_m AS table_m) first forces an incompatible old value struct into the new + // layout. Keep such predicates at table level, after TableReader materializes evolution. + *can_localize = false; + return expr; + } + return cast_file_expr_to_table_type(file_slot, rewrite_info.table_type, rewrite_context); +} + static VExprSPtr rewrite_table_expr_to_file_expr( const VExprSPtr& expr, const std::map& global_to_file_slot, @@ -896,52 +1128,18 @@ static VExprSPtr rewrite_table_expr_to_file_expr( if (rewrite_in_slot_literal_predicate(expr, global_to_file_slot, rewrite_context)) { return expr; } - if (is_struct_element_expr(expr)) { - if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, - rewrite_context)) { - // The scanner still evaluates the original table-level conjunct after TableReader - // finalizes the output block. Skipping an unlocalizable file conjunct is therefore - // safer than preparing a partially rewritten expression against the wrong struct - // layout. In particular, do not generate file-local conjuncts for computed complex - // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct - // slot-rooted struct chains are supported here. - *can_localize = false; - } + if (rewrite_binary_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } - if (expr->is_slot_ref()) { - const auto* slot_ref = assert_cast(expr.get()); - const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); - if (rewrite_it != global_to_file_slot.end()) { - const auto& rewrite_info = rewrite_it->second; - auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); - if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { - return file_slot; - } - if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { - // Generic file-local expressions cannot safely cast an evolved complex file slot - // back to the table type. Example: - // - // table filter: ARRAY_CONTAINS(MAP_KEYS(m), 'person5') - // old file: m MAP> - // table: m MAP> - // - // Although MAP_KEYS only reads the key column, wrapping the file slot as - // `CAST(file_m AS table_m)` forces the value struct cast first and fails because - // the old and new value structs have different fields. Keep such filters at the - // table level, where TableReader materializes the evolved complex value before - // Scanner evaluates the original conjunct. Direct slot-rooted struct child paths - // are handled by rewrite_struct_element_path_to_file_expr() above. - *can_localize = false; - return expr; - } - auto cast_expr = Cast::create_shared(rewrite_info.table_type); - cast_expr->add_child(std::move(file_slot)); - rewrite_context->add_created_expr(cast_expr); - return cast_expr; - } + if (rewrite_in_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } + if (is_struct_element_expr(expr) || expr->is_slot_ref()) { + return rewrite_struct_or_slot_expr_to_file_expr(expr, global_to_file_slot, filter_mappings, + rewrite_context, can_localize); + } // The input is a split-local cloned tree. A previous split-local clone may already have // inserted Cast(slot). Keep that rewrite idempotent: rewrite the cast child from table slot to // the current split's file slot, and drop the cast when the current split no longer needs it. diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index 4f5b247c791efd..b2603a17958eae 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -62,7 +62,14 @@ std::string FileScanRequest::debug_string() const { out << column_id << ":" << block_position; } out << "}, conjunct_count=" << conjuncts.size() - << ", delete_conjunct_count=" << delete_conjuncts.size() << "}"; + << ", delete_conjunct_count=" << delete_conjuncts.size() + << ", count_star_placeholder_columns={"; + const char* delimiter = ""; + for (const auto column_id : count_star_placeholder_columns) { + out << delimiter << column_id.value(); + delimiter = ","; + } + out << "}}"; return out.str(); } diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 59f684121ce1e3..96d940702486b4 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -77,6 +77,18 @@ struct FileScanRequest { // Delete predicates converted to file-local expressions. A TRUE result means that the row is // deleted, so readers must invert each result when building their keep filter. VExprContextSPtrs delete_conjuncts; + // File-local ids retained only because Nereids keeps a minimum-width output tuple for an + // explicit COUNT(*). These columns have no semantic value: for example, after pruning a scan + // may retain an unsupported TIME_MILLIS leaf even though COUNT(*) only needs one row per + // surviving input row. A reader may synthesize defaults instead of reading a marked column + // while it remains non-predicate. If filters or equality deletes promote the same id to + // predicate_columns, the value is semantically required and must still be validated and read. + std::vector count_star_placeholder_columns; + + bool is_count_star_placeholder(LocalColumnId column_id) const { + return std::ranges::find(count_star_placeholder_columns, column_id) != + count_star_placeholder_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 0e348f3de2ccf5..5d0984084a6d41 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -231,6 +231,16 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { return Status::OK(); } + const auto remaining_prefix_bytes = _file_size - _current_offset; + if (remaining_prefix_bytes < sizeof(uint64_t)) { + // Header-only is the one valid empty-file representation and returned above. Once any + // prefix byte exists, all eight bytes are mandatory. For example, `header + 4 bytes` is a + // truncated Native file, not EOF that FileScannerV2 may skip as an empty split. + return Status::InternalError( + "truncated native block length in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, sizeof(uint64_t), remaining_prefix_bytes); + } + uint64_t block_len = 0; Slice len_slice(reinterpret_cast(&block_len), sizeof(block_len)); size_t bytes_read = 0; @@ -247,8 +257,22 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { } _current_offset += sizeof(block_len); if (block_len == 0) { - *eof = (_current_offset >= _file_size); - return Status::OK(); + // A header-only file reaches the `_current_offset >= _file_size` branch above and is a + // valid empty Native file. Once an explicit length prefix is present, however, zero is not + // an EOF marker in the Native format. For example, `header + uint64(0)` is truncated input, + // not a zero-row split, and must not escape as EOF for FileScannerV2 to count as empty. + return Status::InternalError("zero-length native block in file {} at offset {}", + _file_description->path, _current_offset - sizeof(block_len)); + } + + const auto remaining_block_bytes = cast_set(_file_size - _current_offset); + if (block_len > remaining_block_bytes) { + // Validate before allocating `block_len` bytes. Besides reporting a precise corruption + // error, this prevents a truncated file with a damaged length prefix from requesting an + // allocation much larger than the physical file. + return Status::InternalError( + "truncated native block body in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, block_len, remaining_block_bytes); } buffer->assign(block_len, '\0'); diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index b42d47987a54cb..1cdfed80bd273b 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -358,11 +358,16 @@ Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, } column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); column_schema->type = column_schema->type_descriptor.doris_type; + if (column_schema->type == nullptr && + !column_schema->type_descriptor.unsupported_reason.empty()) { + // Keep unsupported logical leaves in the file schema using their physical storage + // type. For example, a file `{id: INT32, clock: TIME_MILLIS}` remains readable for + // `SELECT id`: schema mapping sees `clock` as its physical INT32 but never creates its + // reader. `SELECT clock` still fails explicitly in ParquetColumnReaderFactory before + // any physical value is decoded, preserving the unsupported-type contract. + column_schema->type = column_schema->type_descriptor.physical_doris_type; + } if (column_schema->type == nullptr) { - if (!column_schema->type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", node.name(), - column_schema->type_descriptor.unsupported_reason); - } return Status::NotSupported("Unsupported parquet column type for column {}", node.name()); } diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 52151c7942af2b..aa8f2622ade43b 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "common/check.h" @@ -45,6 +44,75 @@ namespace doris::format::parquet { namespace detail { +namespace { + +bool page_cache_range_less(const ParquetPageCacheRange& lhs, const ParquetPageCacheRange& rhs) { + return lhs.offset < rhs.offset || (lhs.offset == rhs.offset && lhs.size < rhs.size); +} + +} // namespace + +ParquetPageCacheRangeIndex::ParquetPageCacheRangeIndex(size_t max_ranges) + : _max_ranges(max_ranges) { + DORIS_CHECK(_max_ranges > 0); +} + +void ParquetPageCacheRangeIndex::insert(ParquetPageCacheRange range) { + std::lock_guard lock(_mutex); + auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); + if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { + return; + } + if (_ranges.size() >= _max_ranges) { + _ranges.erase(_ranges.begin()); + it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); + } + _ranges.insert(it, range); +} + +void ParquetPageCacheRangeIndex::erase(ParquetPageCacheRange range) { + std::lock_guard lock(_mutex); + const auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); + if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { + _ranges.erase(it); + } +} + +std::vector ParquetPageCacheRangeIndex::ranges() const { + std::lock_guard lock(_mutex); + return _ranges; +} + +size_t ParquetPageCacheRangeIndex::size() const { + std::lock_guard lock(_mutex); + return _ranges.size(); +} + +ParquetPageCacheRangeDirectory::ParquetPageCacheRangeDirectory(size_t max_files) + : _max_files(max_files) { + DORIS_CHECK(_max_files > 0); +} + +std::shared_ptr ParquetPageCacheRangeDirectory::get_or_create( + const std::string& file_key) { + DORIS_CHECK(!file_key.empty()); + std::lock_guard lock(_mutex); + if (const auto it = _indexes.find(file_key); it != _indexes.end()) { + return it->second; + } + if (_indexes.size() >= _max_files) { + _indexes.erase(_indexes.begin()); + } + auto index = std::make_shared(); + _indexes.emplace(file_key, index); + return index; +} + +size_t ParquetPageCacheRangeDirectory::size() const { + std::lock_guard lock(_mutex); + return _indexes.size(); +} + std::vector plan_page_cache_range_read( int64_t position, int64_t nbytes, const std::vector& cached_ranges) { if (position < 0 || nbytes <= 0) { @@ -133,59 +201,14 @@ bool should_use_merge_range_reader(const std::vector& ran namespace { -// StoragePageCache only supports exact-key lookup. Keep lightweight range metadata here so later -// Arrow ReadAt requests can reuse cached bytes when their requested ranges are subsets of, or are -// fully covered by, previously cached ranges. Stale metadata is pruned on lookup. -std::mutex cached_page_range_index_mutex; -std::unordered_map> cached_page_range_index; -constexpr size_t MAX_CACHED_PAGE_RANGE_FILES = 4096; -constexpr size_t MAX_CACHED_PAGE_RANGES_PER_FILE = 65536; - -void register_cached_page_range(const std::string& file_key, int64_t position, int64_t nbytes) { - DORIS_CHECK(nbytes > 0); - std::lock_guard lock(cached_page_range_index_mutex); - if (cached_page_range_index.find(file_key) == cached_page_range_index.end() && - cached_page_range_index.size() >= MAX_CACHED_PAGE_RANGE_FILES) { - cached_page_range_index.erase(cached_page_range_index.begin()); - } - auto& ranges = cached_page_range_index[file_key]; - auto it = std::find_if(ranges.begin(), ranges.end(), [&](const ParquetPageCacheRange& range) { - return range.offset == position && range.size == nbytes; - }); - if (it == ranges.end()) { - if (ranges.size() >= MAX_CACHED_PAGE_RANGES_PER_FILE) { - ranges.erase(ranges.begin()); - } - ranges.push_back(ParquetPageCacheRange {position, nbytes}); - } -} - -void unregister_cached_page_range(const std::string& file_key, - const ParquetPageCacheRange& stale_range) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return; - } - auto& ranges = it->second; - ranges.erase(std::remove_if(ranges.begin(), ranges.end(), - [&](const ParquetPageCacheRange& range) { - return range.offset == stale_range.offset && - range.size == stale_range.size; - }), - ranges.end()); - if (ranges.empty()) { - cached_page_range_index.erase(it); - } -} - -std::vector cached_page_ranges_for_file(const std::string& file_key) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return {}; - } - return it->second; +detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { + // Directory lookup is paid once when a reader opens. ReadAt() then synchronizes only on the + // shared index for this file, so unrelated Parquet files no longer serialize on a process-wide + // hot-path mutex. Strong references deliberately keep range discovery alive after reader A + // closes: reader B can reuse cached [100, 200) for a request [120, 150). The directory and each + // per-file index are independently capped, bounding stale metadata left by page-cache eviction. + static detail::ParquetPageCacheRangeDirectory directory; + return directory; } std::string build_page_cache_file_key(const io::FileReader& file_reader, @@ -215,7 +238,11 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { _base_file_reader(_file_reader), _io_ctx(io_ctx), _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)) { + _page_cache_file_key(std::move(page_cache_file_key)), + _cached_page_range_index( + _enable_page_cache && !_page_cache_file_key.empty() + ? cached_page_range_directory().get_or_create(_page_cache_file_key) + : nullptr) { DORIS_CHECK(_file_reader != nullptr); if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { _file_reader_stats = tracing_reader->stats(); @@ -228,6 +255,10 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { arrow::Status Close() override { if (!_closed) { collect_active_merge_range_profile(); + std::lock_guard lock(_page_cache_mutex); + // Page payloads and their bounded per-file range index intentionally outlive this + // reader for warm scans. Only reader-specific projected ranges are released here. + std::vector().swap(_page_cache_ranges); _closed = true; } return arrow::Status::OK(); @@ -365,7 +396,8 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { private: bool page_cache_enabled() const { return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty(); + StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty() && + _cached_page_range_index != nullptr; } bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { @@ -393,7 +425,7 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { if (!StoragePageCache::instance()->lookup( page_cache_key(cached_range.offset, cached_range.size), &handle, segment_v2::DATA_PAGE)) { - unregister_cached_page_range(_page_cache_file_key, cached_range); + _cached_page_range_index->erase(cached_range); return false; } Slice cached = handle.data(); @@ -406,8 +438,8 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { } bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read( - position, nbytes, cached_page_ranges_for_file(_page_cache_file_key)); + auto plan = detail::plan_page_cache_range_read(position, nbytes, + _cached_page_range_index->ranges()); if (plan.empty()) { return false; } @@ -458,7 +490,8 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { PageCacheHandle handle; StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, segment_v2::DATA_PAGE); - register_cached_page_range(_page_cache_file_key, position, nbytes); + _cached_page_range_index->insert( + ParquetPageCacheRange {.offset = position, .size = nbytes}); ++_page_cache_stats.write_count; ++_page_cache_stats.compressed_write_count; } @@ -516,6 +549,7 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { std::string _page_cache_file_key; mutable std::mutex _page_cache_mutex; std::vector _page_cache_ranges; + std::shared_ptr _cached_page_range_index; ParquetPageCacheStats _page_cache_stats; }; diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..67e9102139dbf2 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include #include "common/status.h" @@ -66,6 +69,39 @@ struct ParquetPageCacheStats { namespace detail { +class ParquetPageCacheRangeIndex { +public: + static constexpr size_t DEFAULT_MAX_RANGES = 65536; + + explicit ParquetPageCacheRangeIndex(size_t max_ranges = DEFAULT_MAX_RANGES); + + void insert(ParquetPageCacheRange range); + void erase(ParquetPageCacheRange range); + + std::vector ranges() const; + size_t size() const; + +private: + const size_t _max_ranges; + mutable std::mutex _mutex; + std::vector _ranges; +}; + +class ParquetPageCacheRangeDirectory { +public: + static constexpr size_t DEFAULT_MAX_FILES = 4096; + + explicit ParquetPageCacheRangeDirectory(size_t max_files = DEFAULT_MAX_FILES); + + std::shared_ptr get_or_create(const std::string& file_key); + size_t size() const; + +private: + const size_t _max_files; + mutable std::mutex _mutex; + std::unordered_map> _indexes; +}; + // Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of // previously cached entries. // StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 753b3628bfa19b..cd84ab74ef953e 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -114,8 +114,66 @@ void collect_request_leaf_column_ids( collect_scan_column(column); } for (const auto& column : request.non_predicate_columns) { - collect_scan_column(column); + if (!request.is_count_star_placeholder(column.column_id())) { + collect_scan_column(column); + } + } +} + +Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (!column_schema.type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, + column_schema.type_descriptor.unsupported_reason); + } + return Status::OK(); + } + for (const auto& child : column_schema.children) { + DORIS_CHECK(child != nullptr); + RETURN_IF_ERROR(validate_all_projected_leaves_supported(*child)); + } + return Status::OK(); +} + +Status validate_projected_leaves_supported(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE || + projection.project_all_children || projection.children.empty()) { + return validate_all_projected_leaves_supported(column_schema); + } + for (const auto& child_projection : projection.children) { + const auto child_it = + std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { + return child_schema->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != column_schema.children.end()); + RETURN_IF_ERROR(validate_projected_leaves_supported(**child_it, child_projection)); + } + return Status::OK(); +} + +Status validate_requested_columns_supported( + const std::vector>& file_schema, + const format::FileScanRequest& request) { + auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { + const auto local_id = projection.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + return Status::OK(); + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + return validate_projected_leaves_supported(*file_schema[local_id], projection); + }; + for (const auto& column : request.predicate_columns) { + RETURN_IF_ERROR(validate_scan_column(column)); + } + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + RETURN_IF_ERROR(validate_scan_column(column)); + } } + return Status::OK(); } std::vector build_page_cache_ranges( @@ -454,6 +512,12 @@ Status ParquetReader::open(std::shared_ptr request) { DORIS_CHECK(local_id >= 0 && local_id < num_fields); } + // Reject requested unsupported logical leaves before row-group statistics, dictionaries, + // bloom filters or page indexes inspect their physical fallback type. For example, a predicate + // on TIME_MILLIS must fail here even when its INT32 statistics would prune every row group; + // otherwise the same unsupported SELECT could fail or silently succeed depending on data. + RETURN_IF_ERROR(validate_requested_columns_supported(_state->file_schema, *request_snapshot)); + RowGroupScanPlan row_group_plan; ParquetScanRange scan_range; scan_range.start_offset = _file_description->range_start_offset; @@ -598,6 +662,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + for (const auto& aggregate_column : request.columns) { + const auto local_id = aggregate_column.projection.local_id(); + if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { + return Status::InvalidArgument("Invalid parquet aggregate column id {}", local_id); + } + DORIS_CHECK(_state->file_schema[local_id] != nullptr); + // Aggregate pushdown can return directly from footer statistics without constructing a + // column reader. Validate first so MIN/MAX(TIME_MILLIS), or an all-pruned COUNT request, + // cannot expose the physical INT32 fallback as a supported logical value. + RETURN_IF_ERROR(validate_projected_leaves_supported(*_state->file_schema[local_id], + aggregate_column.projection)); + } + // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { auto row_group_metadata = @@ -614,6 +691,16 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } const auto& count_projection = request.columns[0].projection; const auto& root_schema = projected_root_schema(_state->file_schema, count_projection); + // A required primitive COUNT(col) still carries its projection so the unsupported-type + // validation above cannot be bypassed. Once validated, its definition level proves that + // every selected row is non-NULL, so preserve the already-computed footer row count and + // avoid reading definition levels merely to rediscover COUNT(col) == COUNT(*). Complex + // roots continue through the shape reader because their count semantics and read-row + // accounting are derived from nested levels. + if (root_schema.kind == ParquetColumnSchemaKind::PRIMITIVE && + root_schema.max_definition_level == 0) { + return Status::OK(); + } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { std::shared_ptr<::parquet::RowGroupReader> row_group; diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 1abddc777aa442..eafb741a53161d 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -177,11 +177,41 @@ std::vector request_scan_columns(const format::FileSca scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); scan_columns.insert(scan_columns.end(), request.predicate_columns.begin(), request.predicate_columns.end()); - scan_columns.insert(scan_columns.end(), request.non_predicate_columns.begin(), - request.non_predicate_columns.end()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + scan_columns.push_back(column); + } + } return scan_columns; } +std::vector physical_non_predicate_columns( + const format::FileScanRequest& request) { + std::vector columns; + columns.reserve(request.non_predicate_columns.size()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + columns.push_back(column); + } + } + return columns; +} + +void materialize_count_star_placeholders(const format::FileScanRequest& request, size_t rows, + Block* file_block) { + DORIS_CHECK(file_block != nullptr); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + continue; + } + const auto block_position = request.local_positions.at(column.column_id()).value(); + auto placeholder = file_block->get_by_position(block_position).column->assert_mutable(); + DCHECK(placeholder->empty()); + placeholder->insert_many_defaults(rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } +} + std::vector build_row_group_prefetch_ranges( const ::parquet::FileMetaData& metadata, const std::vector>& file_schema, @@ -750,6 +780,9 @@ Status ParquetScanScheduler::open_next_row_group( } for (const auto& col : request.non_predicate_columns) { const auto local_id = col.local_id(); + if (request.is_count_star_placeholder(col.column_id())) { + continue; + } if (local_id == format::ROW_POSITION_COLUMN_ID) { _current_non_predicate_columns[local_id] = column_reader_factory.create_row_position_column_reader( @@ -775,7 +808,8 @@ Status ParquetScanScheduler::open_next_row_group( // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, + prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), &_current_non_predicate_prefetched); } *has_row_group = true; @@ -1374,6 +1408,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( _raw_rows_read += batch_rows; if (_current_predicate_columns.empty() && _current_non_predicate_columns.empty()) { *rows = static_cast(batch_rows); + materialize_count_star_placeholders(request, *rows, file_block); if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, batch_rows); } @@ -1431,7 +1466,8 @@ Status ParquetScanScheduler::read_current_row_group_batch( // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, + prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), &_current_non_predicate_prefetched); } @@ -1472,6 +1508,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( file_block->replace_by_position(block_position, std::move(column)); } } + materialize_count_star_placeholders(request, selected_rows, file_block); *rows = static_cast(selected_rows); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index d35181d0397178..8411d53f0fba91 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -293,6 +293,7 @@ ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* co result.physical_type = column->physical_type(); result.converted_type = column->converted_type(); result.fixed_length = column->type_length(); + result.physical_doris_type = physical_type_to_doris_type(column); if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { result.doris_type = logical_type; @@ -306,7 +307,7 @@ ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* co result.doris_type = nullptr; result.supports_record_reader = false; } else { - result.doris_type = physical_type_to_doris_type(column); + result.doris_type = result.physical_doris_type; if (result.physical_type == ::parquet::Type::INT96) { result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index 5d21aae6bae092..a4d99abc0e982a 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -54,6 +54,9 @@ enum class ParquetTimeUnit { // ============================================================================ struct ParquetTypeDescriptor { DataTypePtr doris_type; + // Physical fallback used only to keep file schema construction alive when the logical type is + // unsupported. Column reader creation still rejects unsupported_reason before decoding. + DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 4294d51d043d9e..6c8917d867074c 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -81,6 +81,11 @@ bool HudiHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool HudiHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status HudiHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -145,6 +150,7 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index e22c6bd866f061..77a71cd79ba166 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -60,6 +60,7 @@ class HudiHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 7af551dcddb730..3791a1bb7a7436 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -568,14 +568,13 @@ void IcebergTableReader::_append_equality_delete_row_count_carrier( DORIS_CHECK(request != nullptr); // Columnar readers establish a filter batch's row count from predicate columns. If all // equality keys are missing, the predicate consists only of NULL literals and the filter block - // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; - // normal final materialization ignores this hidden dependency. - const auto carrier_it = std::ranges::find_if( - _data_reader.file_schema, [](const format::ColumnDefinition& field) { - return field.column_type == format::ColumnType::DATA_COLUMN; - }); - DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); - _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + // would otherwise have zero rows. Use the virtual row-position column as the carrier instead + // of an arbitrary physical column. For example, a data file may start with an unsupported + // TIME_MILLIS leaf while the query projects only a supported `id`; selecting that TIME leaf as + // a hidden carrier would make Parquet reject a column the query never requested. Row position + // has one value per input row in both Parquet and ORC, is already used by delete predicates, + // and is explicitly excluded from physical logical-type validation. + _append_file_scan_column(request, format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), &request->predicate_columns); } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index de4562b04088a6..f47d2a18fe7ac7 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -106,6 +106,11 @@ bool PaimonHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool PaimonHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status PaimonHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -173,6 +178,7 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index efaf29d5c76059..66f96dc7898c11 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -66,6 +66,7 @@ class PaimonHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5112a547df9ea9..40654de69dddeb 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -527,6 +527,7 @@ Status TableReader::init(TableReadOptions&& options) { _scanner_profile = options.scanner_profile; _file_slot_descs = options.file_slot_descs; _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; _projected_columns = std::move(options.projected_columns); @@ -861,6 +862,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _deletion_vector = nullptr; _aggregate_pushdown_tried = false; _remaining_table_level_count = -1; + _current_split_uses_metadata_count = false; _current_reader_reached_eof = false; RETURN_IF_ERROR(_evaluate_partition_prune_conjuncts(options.partition_prune_conjuncts, &_current_split_pruned)); @@ -875,12 +877,18 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { // active and no predicate can arrive later. The metadata path can return several batches for // one split; after its first synthetic batch there is no way to recover the real rows if a // runtime filter arrives before the next scheduler turn. - if (_push_down_agg_type == TPushAggOp::type::COUNT && options.all_runtime_filters_applied && + // Table-level metadata only contains the number of rows; it cannot evaluate an expression or + // the NULL state of a COUNT argument. Require the new FE's explicit empty argument list, which + // means COUNT(*)/COUNT(1). A non-empty list means COUNT(col), while nullopt comes from an old FE + // whose COUNT semantics are unknown during a BE-first rolling upgrade. + if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && + _push_down_count_columns->empty() && options.all_runtime_filters_applied && _conjuncts.empty() && options.current_range.__isset.table_format_params && options.current_range.table_format_params.__isset.table_level_row_count) { DORIS_CHECK(options.current_range.table_format_params.table_level_row_count >= -1); _remaining_table_level_count = options.current_range.table_format_params.table_level_row_count; + _current_split_uses_metadata_count = _is_table_level_count_active(); } if (_is_table_level_count_active()) { return Status::OK(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index f62426e5aa5d9f..b9683de564ec5b 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -135,6 +135,10 @@ struct TableReadOptions { const std::vector* file_slot_descs = nullptr; // Push-down aggregate type. const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; + // Table/global indices of explicit COUNT arguments. nullopt means an old FE did not send the + // semantic argument field, while an explicit empty vector means COUNT(*)/COUNT(1). Keeping + // those states separate prevents a rolling-upgrade plan from being reinterpreted by a new BE. + const std::optional> push_down_count_columns = std::nullopt; // Initial digest of predicates available during scanner open. Scanner-driven splits override it // with SplitReadOptions::condition_cache_digest after collecting late-arrival runtime filters. // A zero digest disables condition cache. @@ -147,7 +151,7 @@ struct SplitReadOptions { // Latest scanner conjuncts rewritten to table/global column indices. Runtime filters may // arrive after TableReader::init(), so scanner-driven splits replace the initial snapshot. // nullopt preserves the initial snapshot for standalone TableReader callers. - std::optional conjuncts; + std::optional conjuncts = std::nullopt; // Independent clones used for partition pruning because evaluation prepares and opens them // against a synthetic partition block before the file reader opens its row-level conjuncts. VExprContextSPtrs partition_prune_conjuncts; @@ -207,6 +211,9 @@ class TableReader { virtual Status prepare_split(const SplitReadOptions& options); virtual bool current_split_pruned() const { return _current_split_pruned; } + virtual bool current_split_uses_metadata_count() const { + return _current_split_uses_metadata_count; + } // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start @@ -220,6 +227,7 @@ class TableReader { } _delete_rows = nullptr; _remaining_table_level_count = -1; + _current_split_uses_metadata_count = false; _current_split_pruned = false; return Status::OK(); } @@ -315,6 +323,7 @@ class TableReader { _current_task.reset(); _current_file_description.reset(); _remaining_table_level_count = -1; + _current_split_uses_metadata_count = false; return Status::OK(); } @@ -398,6 +407,20 @@ class TableReader { RETURN_IF_ERROR(close_current_reader()); return Status::OK(); } + // COUNT(*) has no semantic column argument, but Nereids retains a minimum-width scan slot + // so the scan node still has an output tuple. Record only the current non-predicate file + // columns before table-format hooks add row-position or equality-delete dependencies. This + // marker is independent of aggregate eligibility: with position deletes, for example, + // metadata COUNT must fall back to reading rows, but an arbitrary unsupported TIME_MILLIS + // placeholder still must not be validated or decoded merely to carry the surviving count. + if (_push_down_agg_type == TPushAggOp::type::COUNT && + _push_down_count_columns.has_value() && _push_down_count_columns->empty()) { + file_request->count_star_placeholder_columns.reserve( + file_request->non_predicate_columns.size()); + for (const auto& column : file_request->non_predicate_columns) { + file_request->count_star_placeholder_columns.push_back(column.column_id()); + } + } RETURN_IF_ERROR(customize_file_scan_request(file_request.get())); RETURN_IF_ERROR(_open_local_filter_exprs(*file_request)); _data_reader.file_block_layout.clear(); @@ -948,6 +971,9 @@ class TableReader { RETURN_IF_ERROR(status); RETURN_IF_ERROR( _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + if (_push_down_agg_type == TPushAggOp::type::COUNT) { + _current_split_uses_metadata_count = true; + } *pushed_down = true; RETURN_IF_ERROR(close_current_reader()); return Status::OK(); @@ -983,7 +1009,31 @@ class TableReader { return false; } if (agg_type == TPushAggOp::type::COUNT) { - return true; + // Old FEs do not serialize push_down_count_slot_ids. During the supported BE-first + // rolling upgrade, nullopt therefore means "COUNT semantics are unknown", not + // COUNT(*). Fall back to reading rows until the FE explicitly sends either an empty + // list for COUNT(*) or one slot for COUNT(col). + if (!_push_down_count_columns.has_value()) { + return false; + } + // COUNT(*) needs no column metadata. COUNT(col) currently supports one direct file + // column; multiple COUNT arguments fall back to the normal scan so every upper + // aggregate receives the original rows. + if (_push_down_count_columns->empty()) { + return true; + } + if (_push_down_count_columns->size() != 1) { + return false; + } + const auto& mapping = _push_down_count_mapping(); + // Metadata COUNT skips TableReader's normal materialization path. Only a trivial + // mapping is safe: for example, a nullable Parquet INT mapped to a NOT NULL table + // BIGINT normally needs both an INT->BIGINT cast and nullability validation. Counting + // footer values directly would bypass both operations and could hide invalid data. + return mapping.file_local_id.has_value() && mapping.file_type != nullptr && + mapping.table_type != nullptr && mapping.is_trivial && + mapping.virtual_column_type == TableVirtualColumnType::INVALID && + mapping.default_expr == nullptr; } // For MIN/MAX, only support direct file-to-table column mappings. The two emitted rows // must be enough for the upper MIN/MAX aggregate without evaluating default expressions or @@ -1461,24 +1511,19 @@ class TableReader { request->agg_type = agg_type; request->columns.clear(); if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown historically meant COUNT(*) and therefore carried no columns. For - // complex COUNT(col), materializing the full MAP/LIST/STRUCT value only to test the - // top-level NULL bit can be extremely expensive. When the scan projects exactly one - // directly-mapped complex column, pass that file column to the reader so formats such - // as Parquet can count the column shape from metadata/levels without decoding payload - // values like MAP value strings. Other COUNT cases stay on the existing row-count path - // to avoid changing count(*) semantics. - if (_data_reader.column_mapper->mappings().size() == 1) { - const auto& mapping = _data_reader.column_mapper->mappings()[0]; - if (mapping.file_local_id.has_value() && mapping.file_type != nullptr && - is_complex_type(remove_nullable(mapping.file_type)->get_primitive_type()) && - mapping.virtual_column_type == TableVirtualColumnType::INVALID && - mapping.default_expr == nullptr) { - FileAggregateRequest::Column column; - column.projection = - LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); - request->columns.push_back(std::move(column)); - } + DORIS_CHECK(_push_down_count_columns.has_value()); + // An empty explicit list is the semantic signal for COUNT(*). Do not inspect the + // mapping count: `SELECT COUNT(*) FROM t` may still project one nullable column because + // the planner keeps a placeholder slot. In a 10,000-row file where that arbitrary slot + // has 9,015 non-null values, passing the slot would ask Parquet/ORC metadata for + // COUNT(slot)=9,015 instead of the required row count 10,000. + if (!_push_down_count_columns->empty()) { + const auto& mapping = _push_down_count_mapping(); + DORIS_CHECK(mapping.file_local_id.has_value()); + FileAggregateRequest::Column column; + column.projection = + LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); + request->columns.push_back(std::move(column)); } return Status::OK(); } @@ -1495,6 +1540,18 @@ class TableReader { return Status::OK(); } + const ColumnMapping& _push_down_count_mapping() const { + DORIS_CHECK(_push_down_count_columns.has_value()); + DORIS_CHECK(_push_down_count_columns->size() == 1); + const auto mapping_it = + std::ranges::find(_data_reader.column_mapper->mappings(), + _push_down_count_columns->front(), &ColumnMapping::global_index); + // FileScannerV2 translates FE SlotIds through the same projected-column list used to build + // the mapper, so a missing mapping is an FE/BE contract violation rather than a fallback. + DORIS_CHECK(mapping_it != _data_reader.column_mapper->mappings().end()); + return *mapping_it; + } + Status _materialize_aggregate_pushdown_rows(TPushAggOp::type agg_type, const FileAggregateResult& file_result, Block* block) { @@ -1599,6 +1656,7 @@ class TableReader { const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; + std::optional> _push_down_count_columns; size_t _batch_size = 0; uint64_t _initial_condition_cache_digest = 0; uint64_t _condition_cache_digest = 0; @@ -1612,6 +1670,10 @@ class TableReader { int64_t _condition_cache_hit_count = 0; bool _current_reader_reached_eof = false; int64_t _remaining_table_level_count = -1; + // True only after the active split selects a table-level row-count shortcut or successfully + // materializes COUNT rows from file metadata. FileScannerV2 uses this result, rather than the + // raw aggregate opcode, to keep adaptive batching enabled for normal row-scan fallbacks. + bool _current_split_uses_metadata_count = false; // Snapshot supplied by FileScannerV2 for the active split. It gates every shortcut that emits // irreversible aggregate rows, not only the table-level row-count shortcut in prepare_split(). bool _all_runtime_filters_applied_for_split = true; diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 5f479a8843c830..0f0360874c2dd0 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -95,6 +95,29 @@ TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { return range; } +TEST(FileScannerTest, V1CountPushdownRequiresExplicitCountStarArguments) { + EXPECT_EQ(TPushAggOp::type::COUNT, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {})); + + // A missing field is an old FE plan with unknown COUNT semantics, not COUNT(*). + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::nullopt)); + // V1 cannot evaluate COUNT(col) NULL/CAST semantics before replacing the reader with + // CountReader, so an explicit argument must use the normal scan path. + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {7})); + + // The COUNT argument field must not affect other storage-layer aggregate operations. + EXPECT_EQ(TPushAggOp::type::MINMAX, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::MINMAX, std::nullopt)); +} + +TEST(FileScannerV2Test, AdaptiveBatchSizeRunsForCountFallbackOnly) { + EXPECT_TRUE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, false)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, true)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(false, false)); +} + struct RetryableCloseState { int close_calls = 0; }; @@ -617,6 +640,16 @@ TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_not_found(Status::OK(), true)); } +TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { + EXPECT_TRUE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("empty file"), false)); + // Deletion-vector and Parquet readers also use EOF to unwind an interrupted read. Once either + // scanner stop flag is visible, the same status is no longer evidence of an empty file. + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("stop read."), true)); + EXPECT_FALSE( + FileScannerV2::TEST_should_skip_empty(Status::InternalError("read failed"), false)); + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); +} + // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index d241059d6e8202..1a21538544a8b2 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -28,6 +28,7 @@ #include "common/consts.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_decimal.h" @@ -38,6 +39,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vin_predicate.h" @@ -316,6 +318,40 @@ class TestFunctionExpr final : public VExpr { std::string _expr_name; }; +class ExecutableStructElementExpr final : public VExpr { +public: + explicit ExecutableStructElementExpr(DataTypePtr child_type) + : VExpr(std::move(child_type), false) { + set_node_type(TExprNodeType::FUNCTION_CALL); + TFunctionName fn_name; + fn_name.__set_function_name(_expr_name); + _fn.__set_name(fn_name); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(data_type()); + return Status::OK(); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr struct_column; + // branch-4.1's public execution API still takes a mutable selector even though expression + // implementations receive it as read-only; this test expression only forwards it. + RETURN_IF_ERROR(get_child(0)->execute_column( + context, block, const_cast(selector), count, struct_column)); + const auto& input = assert_cast(*struct_column); + result_column = input.get_column_ptr(0); + return Status::OK(); + } + +private: + const std::string _expr_name = "element_at"; +}; + VExprSPtr table_slot(int slot_id, int column_id, DataTypePtr type, const std::string& name) { return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); } @@ -340,6 +376,40 @@ VExprSPtr element_at(const VExprSPtr& parent, DataTypePtr child_type, return expr; } +VExprSPtr executable_struct_element(const VExprSPtr& parent, DataTypePtr child_type, + const std::string& child_name) { + auto expr = std::make_shared(std::move(child_type)); + expr->add_child(parent); + expr->add_child(literal(str(), Field::create_field(child_name))); + return expr; +} + +VExprSPtr executable_binary_predicate(TExprOpcode::type opcode, const VExprSPtr& left, + const VExprSPtr& right) { + const auto result_type = u8(); + TFunctionName fn_name; + fn_name.__set_function_name(opcode == TExprOpcode::GT ? "gt" : "eq"); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({left->data_type()->to_thrift(), right->data_type()->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + + auto expr = VectorizedFnCall::create_shared(node); + expr->add_child(left); + expr->add_child(right); + return expr; +} + VExprSPtr array_element_at(const VExprSPtr& parent, DataTypePtr child_type, int64_t ordinal) { auto expr = std::make_shared("element_at", std::move(child_type)); expr->add_child(parent); @@ -2994,6 +3064,251 @@ TEST(ColumnMapperScanRequestTest, NestedElementAtConjunctUsesFileChildTypeForRen EXPECT_EQ(localized_parent_type->get_element_name(1), "bb"); } +// Scenario: Iceberg promotes a nested struct leaf from INT to BIGINT while an old file still +// stores INT. Because 15 is exactly representable as INT and every INT survives promotion to +// BIGINT, localize `s.b::BIGINT > 15::BIGINT` as `file_s.b::INT > 15::INT`. Rewriting one literal +// avoids casting every file value while preserving the table predicate exactly. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesExactLiteralToFileType) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_leaf = localized_root->children()[0]; + EXPECT_EQ(localized_leaf->expr_name(), "element_at"); + EXPECT_TRUE(localized_leaf->data_type()->equals(*file_int_type)); + const auto& localized_literal = localized_root->children()[1]; + EXPECT_TRUE(localized_literal->is_literal()); + EXPECT_TRUE(localized_literal->data_type()->equals(*file_int_type)); + + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + MutableColumns children; + children.push_back(std::move(values)); + Block block; + block.insert({ColumnStruct::create(std::move(children)), mapper.mappings()[0].file_type, "s"}); + + auto* conjunct = request.conjuncts[0].get(); + auto status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter result(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, result.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + EXPECT_EQ(result, IColumn::Filter({0, 1})); + conjunct->close(); +} + +// Scenario: an old file allows NULL for a nested leaf that the current table declares required. +// Although every non-NULL INT value and the literal 15 can be promoted to BIGINT exactly, the +// predicate must stay above TableReader. If `file_s.b > 15` ran first for rows [NULL, 20], it would +// discard NULL and prevent table-schema materialization from reporting the nullable-to-required +// contract violation. +TEST_F(ColumnMapperCastTest, + NestedElementAtConjunctStaysTableLevelForNullableFileLeafMappedToRequiredTableLeaf) { + const auto file_nullable_int_type = make_nullable(i32()); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_nullable_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: a narrowing file-to-table cast can produce NULL or an error for values that do not fit +// the table leaf. Evaluating that cast below TableReader can filter those rows before +// _align_column_nullability() validates the required table child. Keep the predicate at table level +// so schema materialization observes every source row first. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctStaysTableLevelForNonLosslessFileToTableCast) { + const auto file_bigint_type = i64(); + const auto table_int_type = i32(); + + auto table_a = field_id_col("a", 11, table_int_type); + auto table_struct = struct_col("s", 10, {table_a}); + auto file_a = field_id_col("a", 11, file_bigint_type, 0); + auto file_struct = struct_col("s", 10, {file_a}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_int_type, "a"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, literal(table_int_type, Field::create_field(1))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: the table literal is outside the old file leaf's INT range. Rewriting +// BIGINT 2147483648 to INT would change the predicate, so keep the literal as BIGINT and cast the +// file leaf instead: `CAST(file_s.b::INT AS BIGINT) = 2147483648::BIGINT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctFallsBackForOutOfRangeLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, + literal(table_bigint_type, Field::create_field(2147483648LL))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_cast = localized_root->children()[0]; + ASSERT_NE(dynamic_cast(localized_cast.get()), nullptr); + EXPECT_TRUE(localized_cast->data_type()->equals(*table_bigint_type)); + ASSERT_EQ(localized_cast->get_num_children(), 1); + EXPECT_EQ(localized_cast->children()[0]->expr_name(), "element_at"); + EXPECT_TRUE(localized_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*table_bigint_type)); +} + +// Scenario: the struct leaf is on the right side of the comparison. Literal localization must not +// depend on operand order: `15::BIGINT > s.b::BIGINT` becomes `15::INT > file_s.b::INT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesReverseComparisonLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, literal(table_bigint_type, Field::create_field(15)), + table_leaf); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + EXPECT_TRUE(localized_root->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[0]->is_literal()); + EXPECT_EQ(localized_root->children()[1]->expr_name(), "element_at"); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*file_int_type)); +} + +// Scenario: IN uses one probe type for every candidate. All exact literals may move to the INT +// file domain, but one out-of-range literal makes the complete IN predicate fall back to BIGINT. +TEST_F(ColumnMapperCastTest, NestedElementAtInPredicateUsesAllOrNothingLiteralRewrite) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + const auto build_filter = [&](int64_t second_value) { + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto predicate = create_in_predicate(); + predicate->add_child(table_leaf); + predicate->add_child(literal(table_bigint_type, Field::create_field(10))); + predicate->add_child( + literal(table_bigint_type, Field::create_field(second_value))); + return TableFilter {.conjunct = VExprContext::create_shared(predicate), + .global_indices = {GlobalIndex(0)}}; + }; + + TableColumnMapper exact_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(exact_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest exact_request; + ASSERT_TRUE( + exact_mapper + .create_scan_request({build_filter(20)}, {table_struct}, &exact_request, &state) + .ok()); + ASSERT_EQ(exact_request.conjuncts.size(), 1); + const auto& exact_root = exact_request.conjuncts[0]->root(); + EXPECT_EQ(exact_root->children()[0]->expr_name(), "element_at"); + for (const auto& child : exact_root->children()) { + EXPECT_TRUE(child->data_type()->equals(*file_int_type)); + } + + TableColumnMapper fallback_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(fallback_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest fallback_request; + ASSERT_TRUE(fallback_mapper + .create_scan_request({build_filter(2147483648LL)}, {table_struct}, + &fallback_request, &state) + .ok()); + ASSERT_EQ(fallback_request.conjuncts.size(), 1); + const auto& fallback_root = fallback_request.conjuncts[0]->root(); + const auto& fallback_cast = fallback_root->children()[0]; + ASSERT_NE(dynamic_cast(fallback_cast.get()), nullptr); + EXPECT_TRUE(fallback_cast->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(fallback_root->children()[1]->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_root->children()[2]->data_type()->equals(*table_bigint_type)); +} + // Scenario: output projection reads one struct child while the row filter reads a different nested // struct child. File-local conjunct rewrite must use the merged scan projection type. In the SQL // shape below, `SELECT element_at(s, 'c') WHERE element_at(element_at(s, 'b'), 'cc') LIKE ...` diff --git a/be/test/format_v2/native/native_reader_test.cpp b/be/test/format_v2/native/native_reader_test.cpp index aaa7aa90e0681e..2745ecf852c38b 100644 --- a/be/test/format_v2/native/native_reader_test.cpp +++ b/be/test/format_v2/native/native_reader_test.cpp @@ -295,7 +295,7 @@ TEST(NativeV2ReaderTest, RejectsInvalidHeaderAndEmptyFile) { static_cast(io::global_local_filesystem()->delete_file(empty_path)); } -TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { +TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndReportsHeaderOnlyFileAsEmpty) { std::filesystem::create_directories("./log"); RuntimeState state; RuntimeProfile profile("native_v2_reader_header_boundary_test"); @@ -322,7 +322,8 @@ TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { auto header_only_reader = create_reader(header_only_path, &state, &profile); ASSERT_TRUE(header_only_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(header_only_reader->get_schema(&schema).ok()); + const auto header_only_status = header_only_reader->get_schema(&schema); + EXPECT_TRUE(header_only_status.is()) << header_only_status; static_cast(io::global_local_filesystem()->delete_file(header_only_path)); } @@ -350,6 +351,34 @@ TEST(NativeV2ReaderTest, RejectsTruncatedBlockDuringSchemaProbe) { static_cast(io::global_local_filesystem()->delete_file(path)); } +TEST(NativeV2ReaderTest, RejectsPartialBlockLengthAsCorruption) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_partial_length_test"); + + std::string header; + header.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + header.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + + for (size_t prefix_bytes = 1; prefix_bytes < sizeof(uint64_t); ++prefix_bytes) { + const auto path = "./log/native_v2_partial_length_" + std::to_string(prefix_bytes) + "_" + + UniqueId::gen_uid().to_string() + ".native"; + auto content = header; + content.append(prefix_bytes, '\0'); + ASSERT_TRUE(write_file(path, content).ok()); + + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + const auto status = reader->get_schema(&schema); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("truncated native block length"), std::string::npos); + static_cast(io::global_local_filesystem()->delete_file(path)); + } +} + TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { std::filesystem::create_directories("./log"); RuntimeState state; @@ -374,7 +403,9 @@ TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { auto zero_len_reader = create_reader(zero_len_path, &state, &profile); ASSERT_TRUE(zero_len_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(zero_len_reader->get_schema(&schema).ok()); + const auto zero_len_status = zero_len_reader->get_schema(&schema); + EXPECT_TRUE(zero_len_status.is()) << zero_len_status; + EXPECT_NE(zero_len_status.to_string().find("zero-length native block"), std::string::npos); static_cast(io::global_local_filesystem()->delete_file(zero_len_path)); const auto invalid_pblock_path = diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index fac89b31c422d8..940f94a373a013 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -115,6 +115,45 @@ TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); } +TEST(ParquetPageCacheRangeTest, PerFileRangeIndexDeduplicatesAndEvictsAtCapacity) { + detail::ParquetPageCacheRangeIndex index(3); + index.insert({200, 20}); + index.insert({100, 30}); + index.insert({100, 10}); + index.insert({100, 30}); + + EXPECT_EQ(index.size(), 3); + index.insert({300, 40}); + const auto ranges = index.ranges(); + ASSERT_EQ(ranges.size(), 3); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 30); + EXPECT_EQ(ranges[1].offset, 200); + EXPECT_EQ(ranges[1].size, 20); + EXPECT_EQ(ranges[2].offset, 300); + + index.erase({100, 30}); + EXPECT_EQ(index.size(), 2); +} + +TEST(ParquetPageCacheRangeTest, DirectorySharesBoundedIndexAcrossReaderLifetimes) { + detail::ParquetPageCacheRangeDirectory directory(2); + auto first_reader_index = directory.get_or_create("file-a"); + first_reader_index->insert({100, 100}); + first_reader_index.reset(); + + // The directory owns the per-file index, so reader B still discovers reader A's wider cache + // entry and can plan a subset hit after A closes. + auto second_reader_index = directory.get_or_create("file-a"); + const auto plan = detail::plan_page_cache_range_read(120, 30, second_reader_index->ranges()); + ASSERT_EQ(plan.size(), 1); + expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); + + directory.get_or_create("file-b"); + directory.get_or_create("file-c"); + EXPECT_EQ(directory.size(), 2); +} + TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 2fd85cefd49ac9..36b7cebdaa9cb9 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -1206,6 +1206,20 @@ TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordRea EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); } +TEST(ParquetColumnReaderFactoryTest, RejectsProjectedUnsupportedLogicalType) { + ParquetColumnSchema schema = int64_schema("unsupported_time"); + schema.kind = ParquetColumnSchemaKind::PRIMITIVE; + schema.type_descriptor.unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; + + ParquetColumnReaderFactory factory(nullptr, 1); + std::unique_ptr reader; + const auto status = factory.create(schema, &reader); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(schema.type_descriptor.unsupported_reason), + std::string::npos); +} + TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { auto schema = struct_schema_for_projection(); ParquetColumnReaderFactory factory(nullptr, 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 55387d2c1ed380..36cc99ebf106c0 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,8 @@ #include "core/field.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" @@ -304,6 +307,32 @@ void write_table(const std::string& file_path, const std::shared_ptr out = *file_result; + + // Arrow intentionally writes time32 as local time (isAdjustedToUTC=false), so use Parquet's + // low-level writer to produce the unsupported required logical type needed by this aggregate + // regression. Required is important: Nereids may push COUNT(col) because NULL filtering cannot + // change its value, which is precisely the path where an empty aggregate request lost `col`. + const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {adjusted_time}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + const int32_t values[] = {1000, 2000}; + EXPECT_EQ(time_writer->WriteBatch(2, nullptr, nullptr, values), 2); + time_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group_size = 2, bool enable_statistics = true) { auto schema = arrow::schema({ @@ -737,6 +766,15 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(count_result.count, 6); EXPECT_TRUE(count_result.columns.empty()); + format::FileAggregateResult required_count_result; + format::FileAggregateRequest required_count_request; + required_count_request.agg_type = TPushAggOp::COUNT; + required_count_request.columns.push_back({.projection = field_projection(0)}); + ASSERT_TRUE(reader->get_aggregate_result(required_count_request, &required_count_result).ok()); + // The required scalar projection is retained for logical-type validation, but after that + // validation COUNT(id) can reuse the same selected-row-group footer count as COUNT(*). + EXPECT_EQ(required_count_result.count, 6); + format::FileAggregateResult minmax_result; format::FileAggregateRequest minmax_request; minmax_request.agg_type = TPushAggOp::MINMAX; @@ -753,6 +791,96 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(minmax_result.columns[1].max_value.get(), 60); } +TEST_F(ParquetScanTest, AggregateCountRejectsRequiredUnsupportedScalarProjection) { + write_required_adjusted_time_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::COUNT; + request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + const auto status = reader->get_aggregate_result(request, &result); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +TEST_F(ParquetScanTest, CountStarIgnoresUnsupportedPlannerPlaceholder) { + write_required_adjusted_time_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + // A normal projection still requests the TIME_MILLIS value and must fail at open, before + // row-group pruning or physical INT32 statistics can hide the unsupported logical type. + auto projected_reader = create_reader(); + ASSERT_TRUE(projected_reader->init(&state).ok()); + auto projected_request = std::make_shared(); + format::FileScanRequestBuilder projected_builder(projected_request.get()); + ASSERT_TRUE(projected_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + const auto projected_status = projected_reader->open(projected_request); + EXPECT_TRUE(projected_status.is()) << projected_status; + + // Metadata COUNT(*) carries the same retained scan slot, but its aggregate request has no + // semantic column. The placeholder must not make open fail before footer row counts are used. + auto metadata_count_reader = create_reader(); + ASSERT_TRUE(metadata_count_reader->init(&state).ok()); + auto metadata_count_request = std::make_shared(); + format::FileScanRequestBuilder metadata_count_builder(metadata_count_request.get()); + ASSERT_TRUE(metadata_count_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + metadata_count_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + ASSERT_TRUE(metadata_count_reader->open(metadata_count_request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + format::FileAggregateResult aggregate_result; + ASSERT_TRUE( + metadata_count_reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); + + // The marker is independent of aggregate eligibility. Simulate a position delete, + // which disables metadata COUNT and requires the reader to produce surviving rows. Parquet + // reads only the virtual row position, filters row 1, and fills a default placeholder for row 0 + // without validating or decoding either unsupported TIME_MILLIS value. + auto count_star_reader = create_reader(); + ASSERT_TRUE(count_star_reader->init(&state).ok()); + auto count_star_request = std::make_shared(); + format::FileScanRequestBuilder count_star_builder(count_star_request.get()); + ASSERT_TRUE(count_star_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(count_star_builder + .add_predicate_column(format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) + .ok()); + count_star_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + + const std::vector deleted_rows {1}; + auto delete_predicate = std::make_shared(deleted_rows); + const auto row_position = count_star_request->local_positions.at( + format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)); + delete_predicate->add_child(VSlotRef::create_shared( + cast_set(row_position.value()), cast_set(row_position.value()), -1, + std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + auto delete_context = VExprContext::create_shared(std::move(delete_predicate)); + ASSERT_TRUE(delete_context->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(delete_context->open(&state).ok()); + count_star_request->delete_conjuncts.push_back(std::move(delete_context)); + ASSERT_TRUE(count_star_reader->open(count_star_request).ok()); + + std::vector file_schema; + ASSERT_TRUE(count_star_reader->get_schema(&file_schema).ok()); + file_schema.push_back(format::row_position_column_definition()); + auto block = build_file_block(file_schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(count_star_reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(block.get_by_position(0).column->size(), 1); + const auto& positions = + assert_cast(*block.get_by_position(row_position.value()).column); + ASSERT_EQ(positions.size(), 1); + EXPECT_EQ(positions.get_element(0), 0); +} + TEST_F(ParquetScanTest, AggregateMinMaxRejectsInexactBinaryStatistics) { write_binary_minmax_parquet_file(_file_path); auto reader = create_reader(); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index e620ed718efbf2..9f78735bfb560a 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -414,7 +414,7 @@ TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { EXPECT_TRUE(fields.empty()); } -TEST(ParquetSchemaTest, RejectInvalidListMapAndUnsupportedTime) { +TEST(ParquetSchemaTest, RejectInvalidListMapAndPreserveUnsupportedTime) { const auto bad_list = ::parquet::schema::GroupNode::Make( "bad_list", ::parquet::Repetition::OPTIONAL, {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, @@ -432,9 +432,11 @@ TEST(ParquetSchemaTest, RejectInvalidListMapAndUnsupportedTime) { const auto converted_time = ::parquet::schema::PrimitiveNode::Make( "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, ::parquet::ConvertedType::TIME_MILLIS); - const auto status = build_status({converted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + const auto fields = build_fields({converted_time}); + ASSERT_EQ(fields.size(), 1); + EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_INT); + EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find( + "Parquet TIME with isAdjustedToUTC=true is not supported"), std::string::npos); } @@ -513,15 +515,21 @@ TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { EXPECT_FALSE(build_status({repeated_map}).ok()); } -TEST(ParquetSchemaTest, LogicalUtcTimeIsRejected) { +TEST(ParquetSchemaTest, LogicalUtcTimeIsPreservedForProjection) { const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), ::parquet::Type::INT32); - const auto status = build_status({adjusted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); + const auto supported_value = ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64); + const auto row = ::parquet::schema::GroupNode::Make("row", ::parquet::Repetition::OPTIONAL, + {adjusted_time, supported_value}); + const auto fields = build_fields({row}); + ASSERT_EQ(fields.size(), 1); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(remove_nullable(fields[0]->children[0]->type)->get_primitive_type(), TYPE_INT); + EXPECT_FALSE(fields[0]->children[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_EQ(remove_nullable(fields[0]->children[1]->type)->get_primitive_type(), TYPE_BIGINT); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 28c396371e4559..2dd001469d8a66 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -17,14 +17,20 @@ #include "format_v2/table/hudi_reader.h" +#include +#include #include +#include +#include +#include #include #include #include #include #include +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" @@ -32,6 +38,9 @@ #include "format_v2/column_data.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" namespace doris::format { namespace { @@ -69,6 +78,39 @@ ColumnDefinition make_file_column(int32_t id, const std::string& name, const Dat return field; } +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = name; + column.type = make_nullable(type); + return column; +} + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +void write_int_parquet_file(const std::string& file_path, const std::vector& values) { + arrow::Int32Builder value_builder; + for (const auto value : values) { + ASSERT_TRUE(value_builder.Append(value).ok()); + } + std::shared_ptr value_array; + ASSERT_TRUE(value_builder.Finish(&value_array).ok()); + const auto table = arrow::Table::Make( + arrow::schema({arrow::field("id", arrow::int32(), false)}), {value_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr output = *file_result; + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + static_cast(values.size()))); +} + TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { TTableFormatFileDesc table_format_params; table_format_params.__set_table_format_type("hudi"); @@ -201,5 +243,62 @@ TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 123); } +TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "base-file.parquet").string(); + write_int_parquet_file(file_path, {1, 2, 3}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + io_ctx->file_cache_stats = &file_cache_stats; + ShardedKVCache cache(1); + + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + split_options.current_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_table_format_params(hudi_table_format_desc(std::nullopt)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index d5ce700970496e..f6bd784b6ecd08 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -449,6 +450,39 @@ void write_single_int_parquet_file(const std::string& file_path, const std::stri builder.build())); } +void write_unsupported_time_first_int_parquet_file(const std::string& file_path, + const std::vector& ids) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + // Arrow writes time32 as isAdjustedToUTC=false, which Doris supports. Use Parquet's low-level + // writer so the first physical column is the deliberately unsupported adjusted TIME_MILLIS + // shape from the review report. The query will project only the following supported `id`. + const auto unsupported_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto id = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {unsupported_time, id}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + std::vector times(ids.size(), 1000); + const auto row_count = cast_set(ids.size()); + EXPECT_EQ(time_writer->WriteBatch(row_count, nullptr, nullptr, times.data()), row_count); + time_writer->Close(); + auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + EXPECT_EQ(id_writer->WriteBatch(row_count, nullptr, nullptr, ids.data()), row_count); + id_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, const std::vector& first_values, std::optional first_field_id, @@ -1900,6 +1934,9 @@ TEST(IcebergV2ReaderTest, IcebergTableLevelCountUsesAssignedRowCountWithPosition .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + // An explicit empty argument list is the new FE marker for + // COUNT(*)/COUNT(1); nullopt intentionally exercises fallback. + .push_down_count_columns = std::vector {}, }) .ok()); @@ -2109,6 +2146,45 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMissingKeyDoesNotReadUnsupportedUnprojectedCarrier) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_virtual_carrier_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_unsupported_time_first_int_parquet_file(file_path, {1, 2, 3}); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // The missing data key is NULL, so it does not match delete key 7. More importantly, the + // hidden row-count dependency must use virtual row position instead of the first physical + // TIME_MILLIS column, which is unsupported and was not requested by this `id` projection. + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_missing_default_test"; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 4d7fb6a75965a8..66e17ab4e687dc 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -661,6 +661,59 @@ TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 321); } +TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "data-file.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 4ee2258d1273c0..1c7bb903ac7b20 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1013,6 +1013,7 @@ struct FakeFileReaderState { bool stop_during_read = false; bool not_found_during_init = false; std::shared_ptr last_request; + std::optional last_aggregate_request; std::shared_ptr condition_cache_ctx; std::shared_ptr io_ctx; }; @@ -1128,6 +1129,7 @@ class FakeFileReader final : public FileReader { if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("fake reader only supports COUNT aggregate pushdown"); } + _state->last_aggregate_request = request; if (_state->stop_during_aggregate) { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; @@ -1511,6 +1513,7 @@ TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1553,6 +1556,7 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1570,6 +1574,10 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_EQ(fake_state->open_count, 1); EXPECT_EQ(block.rows(), 2); + ASSERT_NE(fake_state->last_request, nullptr); + // Aggregate pushdown is disabled while a runtime filter is pending, but COUNT(*) semantics do + // not change. The retained output slot remains a value-less placeholder during row fallback. + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); ASSERT_TRUE(reader.close().ok()); } @@ -1653,6 +1661,7 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1715,11 +1724,12 @@ TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { } TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { + const auto nullable_int_type = make_nullable(std::make_shared()); std::vector file_schema; - file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); std::vector projected_columns; - projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); set_name_identifiers(&projected_columns); io::FileReaderStats file_reader_stats; @@ -1740,6 +1750,8 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); @@ -1754,6 +1766,230 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { EXPECT_EQ(block.rows(), 3); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(fake_state->last_request != nullptr); + EXPECT_TRUE(fake_state->last_request->count_star_placeholder_columns.empty()); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + ASSERT_EQ(fake_state->last_aggregate_request->columns.size(), 1); + // A primitive COUNT(col) projection must reach the file reader just like a complex one. + EXPECT_EQ(fake_state->last_aggregate_request->columns[0].projection.local_id(), 0); +} + +TEST(TableReaderTest, PushDownCountStarIgnoresProjectedPlaceholderColumn) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "nullable_id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "nullable_id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // COUNT(*) deliberately has no explicit count columns. The + // nullable_id projection is only the planner's scan placeholder. + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(fake_state->last_request != nullptr); + ASSERT_EQ(fake_state->last_request->count_star_placeholder_columns.size(), 1); + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + // Passing nullable_id here would implement COUNT(nullable_id) and reproduce the external ORC + // and Parquet failures where footer row counts were reduced by null values. + EXPECT_TRUE(fake_state->last_aggregate_request->columns.empty()); +} + +TEST(TableReaderTest, PushDownCountFallsBackWhenSemanticArgumentsAreAbsent) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // Simulate an old FE: it can request COUNT pushdown but cannot + // serialize push_down_count_slot_ids. + .push_down_count_columns = std::nullopt, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + // The explicit aggregate_count=3 would be returned if absence were confused with COUNT(*). + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); +} + +TEST(TableReaderTest, PushDownCountFallsBackForMultipleArguments) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_string_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + file_schema.push_back(make_file_column(1, "name", nullable_string_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + projected_columns.push_back(make_table_column(1, "name", nullable_string_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0), GlobalIndex(1)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); +} + +TEST(TableReaderTest, PushDownCountFallsBackForNullableToRequiredMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + // make_table_column models the usual nullable external-table descriptor. Override it here to + // reproduce an evolved table contract that declares the mapped physical column required. + projected_columns[0].type = std::make_shared(); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + // The normal scan rejects the nullable physical column because it cannot satisfy the required + // table contract. Footer COUNT would bypass that validation and incorrectly return 3. + EXPECT_FALSE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); +} + +TEST(TableReaderTest, PushDownCountFallsBackForCastMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_bigint_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(block.get_by_position(0).type->equals(*nullable_bigint_type)); + // INT->BIGINT is a non-trivial mapping, so COUNT must not skip the cast/materialization path. + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); } TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { @@ -1781,6 +2017,7 @@ TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1821,6 +2058,7 @@ TEST(TableReaderTest, DebugStringCoversReaderStateAndEnumNames) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -2467,6 +2705,7 @@ TEST(TableReaderTest, PushDownCountFromNewParquetReader) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -2508,6 +2747,7 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); auto split_options = build_split_options(file_path); @@ -2539,6 +2779,56 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, TableLevelCountRequiresExplicitCountStarArguments) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_table_count_arguments_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + const std::vector>> unsafe_count_arguments { + std::nullopt, std::vector {GlobalIndex(0)}}; + for (const auto& count_arguments : unsafe_count_arguments) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = count_arguments, + }) + .ok()); + auto split_options = build_split_options(file_path); + // Five metadata rows deliberately disagree with the three physical rows. nullopt models an + // old FE, while the non-empty vector models COUNT(id); neither may be treated as COUNT(*). + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + size_t total_rows = 0; + bool eos = false; + while (!eos) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + total_rows += block.rows(); + } + EXPECT_EQ(3, total_rows); + ASSERT_TRUE(reader.close().ok()); + } + + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, PushDownMinMaxFromNewParquetReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_minmax_test"; std::filesystem::remove_all(test_dir); @@ -3355,6 +3645,7 @@ TEST(TableReaderTest, PushDownCountOnlyUsesSelectedRowGroupInFileRange) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 2)).ok()); @@ -3394,6 +3685,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithTableConjunct) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -3435,6 +3727,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithFilter) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index e36ebefe646eb2..f017c444c69122 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -92,6 +92,8 @@ public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, @Override protected void toThrift(TPlanNode planNode) { planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp); + planNode.setPushDownCountSlotIds( + pushDownCountSlotIds.stream().map(id -> id.asInt()).collect(Collectors.toList())); planNode.setNodeType(TPlanNodeType.FILE_SCAN_NODE); TFileScanNode fileScanNode = new TFileScanNode(); @@ -107,6 +109,21 @@ protected void setPushDownCount(long count) { tableLevelRowCount = count; } + /** + * Return whether FE may replace real table-format splits with metadata COUNT splits. + * + *

The aggregate opcode alone is insufficient because both {@code COUNT(*)} and + * {@code COUNT(col)} use {@link TPushAggOp#COUNT}. The semantic argument list distinguishes + * them: it is empty only for {@code COUNT(*)}/{@code COUNT(1)}. For example, if an Iceberg + * table has 100 data files, retaining one representative split is correct for a snapshot + * {@code COUNT(*)}. Doing that for {@code COUNT(required_col)} is unsafe: BE deliberately + * falls back to reading the column, but it would then see only the representative file and + * undercount the table. + */ + protected boolean isTableLevelCountStarPushdown() { + return pushDownAggNoGroupingOp == TPushAggOp.COUNT && pushDownCountSlotIds.isEmpty(); + } + private long getPushDownCount() { return tableLevelRowCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 6c96dab3aa0ea2..f5acd7834b6402 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -59,7 +59,6 @@ import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; import org.apache.doris.thrift.TTransactionalHiveDesc; @@ -336,7 +335,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List getSplits(int numBackends) throws UserException { ++paimonSplitNum; } - boolean applyCountPushdown = getPushDownAggNoGroupingOp() == TPushAggOp.COUNT; + // Merged row counts contain only COUNT(*) semantics. COUNT(col) must keep every DataSplit + // because BE will read the argument column to account for NULL and schema-mapping rules. + boolean applyCountPushdown = isTableLevelCountStarPushdown(); // Used to avoid repeatedly calculating partition info map for the same // partition data. // And for counting the number of selected partitions for this paimon table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 6ba1b5d4a32821..71fbb2fe53dff3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -45,7 +45,6 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.collect.Lists; @@ -141,9 +140,9 @@ public List getSplits(int numBackends) throws UserException { List fileStatuses = tableValuedFunction.getFileStatuses(); - // Push down count optimization. + // Avoid splitting only for table-level COUNT(*). COUNT(column) still reads column data. boolean needSplit = true; - if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT) { + if (isTableLevelCountStarPushdown()) { int parallelNum = sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()); int totalFileNum = fileStatuses.size(); needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index c835e1dfc71e8b..13d895ad7e4e84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -878,6 +878,12 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca scanNode.setNereidsId(fileScan.getId()); context.getNereidsIdToPlanNodeIdMap().put(fileScan.getId(), scanNode.getId()); scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId())); + scanNode.setPushDownCountSlotIds(context.getRelationPushCountArgumentExprIds(fileScan.getRelationId()) + .stream() + .map(exprId -> Objects.requireNonNull(context.findSlotRef(exprId), + "missing slot for pushed-down COUNT argument " + exprId).getSlotId()) + .collect(Collectors.toList())); + scanNode.setHasPartitionPredicate(fileScan.hasPartitionPredicate()); TableNameInfo tableNameInfo = new TableNameInfo(null, "", ""); TableRefInfo ref = new TableRefInfo(tableNameInfo, null, null); @@ -1434,6 +1440,10 @@ public PlanFragment visitPhysicalStorageLayerAggregate( context.setRelationPushAggOp( storageLayerAggregate.getRelation().getRelationId(), pushAggOp); + context.setRelationPushCountArgumentExprIds( + storageLayerAggregate.getRelation().getRelationId(), + pushAggOp == TPushAggOp.COUNT + ? storageLayerAggregate.getCountArgumentExprIds() : ImmutableList.of()); PlanFragment planFragment = storageLayerAggregate.getRelation().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 7420524239d465..55496fa2ac5c98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -55,6 +55,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -113,6 +114,7 @@ public class PlanTranslatorContext { private final Map cteScanNodeMap = Maps.newHashMap(); private final Map tablePushAggOp = Maps.newHashMap(); + private final Map> tablePushCountArgumentExprIds = Maps.newHashMap(); private final Map> statsUnknownColumnsMap = Maps.newHashMap(); private final RuntimeFilterContextV2 runtimeFilterV2Context; @@ -384,6 +386,14 @@ public TPushAggOp getRelationPushAggOp(RelationId relationId) { return tablePushAggOp.getOrDefault(relationId, TPushAggOp.NONE); } + public void setRelationPushCountArgumentExprIds(RelationId relationId, List exprIds) { + tablePushCountArgumentExprIds.put(relationId, Lists.newArrayList(exprIds)); + } + + public List getRelationPushCountArgumentExprIds(RelationId relationId) { + return tablePushCountArgumentExprIds.getOrDefault(relationId, Collections.emptyList()); + } + public boolean isTopMaterializeNode() { return isTopMaterializeNode; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 22356c98e760d6..cd0faffeb5edbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Or; @@ -569,6 +570,7 @@ private LogicalAggregate storageLayerAggregate( Map, PushDownAggOp> supportedAgg = PushDownAggOp.supportedFunctions(); boolean containsCount = false; + boolean countHasCastArgument = false; Set checkNullSlots = new HashSet<>(); Set expressionAfterProject = new HashSet<>(); @@ -590,6 +592,7 @@ private LogicalAggregate storageLayerAggregate( checkNullSlots.add((SlotReference) arg0); expressionAfterProject.add(arg0); } else if (arg0 instanceof Cast) { + countHasCastArgument = true; Expression child0 = arg0.child(0); if (child0 instanceof SlotReference) { checkNullSlots.add((SlotReference) child0); @@ -666,6 +669,7 @@ private LogicalAggregate storageLayerAggregate( return canNotPush; } else { if (needCheckSlotNull) { + countHasCastArgument = true; checkNullSlots.add((SlotReference) argument.child(0)); } } @@ -676,6 +680,16 @@ private LogicalAggregate storageLayerAggregate( argumentsOfAggregateFunction = processedExpressions; } + // File aggregate metadata can describe COUNT(*) or COUNT(file_column), but it cannot + // describe the CAST wrapped around a COUNT argument. Dropping that CAST is incorrect even + // when the source column is NOT NULL. For example, a non-null DOUBLE value outside the INT + // range becomes NULL for CAST(double_col AS INT), so COUNT(CAST(double_col AS INT)) must + // exclude it while a footer-level COUNT(double_col) would include it. Keep OLAP's existing + // storage-layer behavior unchanged, and make external files evaluate the CAST normally. + if (logicalScan instanceof LogicalFileScan && countHasCastArgument) { + return canNotPush; + } + Set pushDownAggOps = functionClasses.stream() .map(supportedAgg::get) .collect(Collectors.toSet()); @@ -689,6 +703,12 @@ private LogicalAggregate storageLayerAggregate( List usedSlotInTable = (List) Project.findProject(aggUsedSlots, logicalScan.getOutput()); + // COUNT(*) has no aggregate arguments, even though later column pruning retains one + // arbitrary scan slot. Preserve the semantic arguments here so the BE never needs to infer + // COUNT(col) from the post-pruning scan shape. + List countArgumentExprIds = mergeOp == PushDownAggOp.COUNT + ? usedSlotInTable.stream().map(SlotReference::getExprId).collect(Collectors.toList()) + : ImmutableList.of(); for (SlotReference slot : usedSlotInTable) { Optional optionalColumn = slot.getOriginalColumn(); @@ -732,11 +752,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } @@ -748,11 +769,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 2385c0601e633f..1d4255b6df98ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -20,10 +20,12 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -42,21 +44,30 @@ public class PhysicalStorageLayerAggregate extends PhysicalCatalogRelation { private final PhysicalCatalogRelation relation; private final PushDownAggOp aggOp; + private final List countArgumentExprIds; public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp) { + this(relation, aggOp, ImmutableList.of()); + } + + public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), Optional.empty(), relation.getLogicalProperties(), ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), groupExpression, logicalProperties, physicalProperties, statistics, ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalRelation getRelation() { @@ -67,6 +78,10 @@ public PushDownAggOp getAggOp() { return aggOp; } + public List getCountArgumentExprIds() { + return countArgumentExprIds; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalStorageLayerAggregate(this, context); @@ -82,29 +97,30 @@ public String toString() { } public PhysicalStorageLayerAggregate withPhysicalOlapScan(PhysicalOlapScan physicalOlapScan) { - return new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp); + return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( + physicalOlapScan, aggOp, countArgumentExprIds)); } @Override public PhysicalStorageLayerAggregate withGroupExpression(Optional groupExpression) { - return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, - getLogicalProperties(), physicalProperties, statistics); + return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, + countArgumentExprIds, groupExpression, getLogicalProperties(), physicalProperties, statistics)); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, - logicalProperties.get(), physicalProperties, statistics); + return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, + countArgumentExprIds, groupExpression, logicalProperties.get(), physicalProperties, statistics)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalStorageLayerAggregate( + return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( (PhysicalCatalogRelation) relation.withPhysicalPropertiesAndStats(null, statistics), - aggOp, groupExpression, - getLogicalProperties(), physicalProperties, statistics); + aggOp, countArgumentExprIds, groupExpression, + getLogicalProperties(), physicalProperties, statistics)); } /** PushAggOp */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index d7e63e6e2bb266..ccf6bf3fd79de9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -866,11 +866,18 @@ public void setCardinalityAfterFilter(long cardinalityAfterFilter) { } protected TPushAggOp pushDownAggNoGroupingOp = TPushAggOp.NONE; + // Explicit COUNT arguments. COUNT(*)/COUNT(1) intentionally keep this empty even though + // column pruning retains one placeholder scan slot. + protected List pushDownCountSlotIds = Collections.emptyList(); public void setPushDownAggNoGrouping(TPushAggOp pushDownAggNoGroupingOp) { this.pushDownAggNoGroupingOp = pushDownAggNoGroupingOp; } + public void setPushDownCountSlotIds(List pushDownCountSlotIds) { + this.pushDownCountSlotIds = Lists.newArrayList(pushDownCountSlotIds); + } + public void setChildrenDistributeExprLists(List> childrenDistributeExprLists) { this.childrenDistributeExprLists = childrenDistributeExprLists; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index ec72503f7a674f..88bdbdd6d1018f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -29,6 +29,7 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TIcebergDeleteFileDesc; +import org.apache.doris.thrift.TPushAggOp; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; @@ -91,6 +92,56 @@ protected boolean getEnableMappingVarbinary() { } } + private static class CountPlanningIcebergScanNode extends IcebergScanNode { + private final TableScan tableScan; + private final long snapshotCount; + private int snapshotCountCalls; + + CountPlanningIcebergScanNode(SessionVariable sv, TableScan tableScan, long snapshotCount) { + super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); + this.tableScan = tableScan; + this.snapshotCount = snapshotCount; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + + @Override + public long getCountFromSnapshot() { + ++snapshotCountCalls; + return snapshotCount; + } + } + + @Test + public void testTableLevelCountSplitPlanningRequiresCountStar() { + SessionVariable sv = Mockito.mock(SessionVariable.class); + Mockito.when(sv.getEnableExternalTableBatchMode()).thenReturn(false); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(tableScan.snapshot()).thenReturn(Mockito.mock(Snapshot.class)); + + // COUNT(required_col) carries a non-empty semantic argument list. Even though its result + // equals COUNT(*) for valid data, BE intentionally reads the column to enforce schema + // contracts. FE must therefore leave all real file tasks available to that fallback. + CountPlanningIcebergScanNode countColumnNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + Assert.assertFalse(countColumnNode.isBatchMode()); + Assert.assertEquals(0, countColumnNode.snapshotCountCalls); + + // COUNT(*) has an explicitly empty argument list, so snapshot row count remains eligible + // and doGetSplits may retain only representative tasks for parallel materialization. + CountPlanningIcebergScanNode countStarNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + Assert.assertFalse(countStarNode.isBatchMode()); + Assert.assertEquals(1, countStarNode.snapshotCountCalls); + } + @Test public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index d31ecb12ee45f1..c15132cdc99deb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.ExceptionChecker; @@ -34,6 +35,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TPushAggOp; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.io.DataFileMeta; @@ -68,6 +70,38 @@ public class PaimonScanNodeTest { @Mock private PaimonFileExternalCatalog paimonFileExternalCatalog; + @Test + public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException { + PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); + node.setSource(mockPaimonSourceWithPartitionKeys(Collections.emptyList())); + List dataSplits = Arrays.asList( + mockCountDataSplit("f1.parquet", 4_000), + mockCountDataSplit("f2.parquet", 5_000), + mockCountDataSplit("f3.parquet", 6_000)); + Mockito.doReturn(dataSplits).when(node).getPaimonSplitFromAPI(); + Mockito.when(sv.isForceJniScanner()).thenReturn(true); + Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE"); + Mockito.when(sv.getParallelExecInstanceNum(ArgumentMatchers.nullable(String.class))).thenReturn(1); + + // Before the fix, the raw COUNT opcode made this path keep only parallel representative + // splits and attach the 15,000 metadata rows. BE rejects that shortcut for COUNT(col), so + // it would scan only those representatives and silently miss the discarded DataSplits. + node.setPushDownAggNoGrouping(TPushAggOp.COUNT); + node.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + List countColumnSplits = node.getSplits(1); + Assert.assertEquals(3, countColumnSplits.size()); + for (org.apache.doris.spi.Split split : countColumnSplits) { + Assert.assertFalse(((PaimonSplit) split).getRowCount().isPresent()); + } + + // COUNT(*) remains metadata-only. The 15,000 rows exceed the parallel threshold, so one + // configured execution instance retains one representative split carrying the full sum. + node.setPushDownCountSlotIds(Collections.emptyList()); + List countStarSplits = node.getSplits(1); + Assert.assertEquals(1, countStarSplits.size()); + Assert.assertEquals(Optional.of(15_000L), ((PaimonSplit) countStarSplits.get(0)).getRowCount()); + } + @Test public void testSplitWeight() throws UserException { @@ -744,4 +778,20 @@ private DataSplit createDataSplit(String fileName) { .withDataFiles(Collections.singletonList(dataFileMeta)) .build(); } + + private DataSplit mockCountDataSplit(String fileName, long rowCount) { + DataFileMeta dataFileMeta = DataFileMeta.forAppend(fileName, 64L * 1024 * 1024, rowCount, + SimpleStats.EMPTY_STATS, 1L, 1L, 1L, Collections.emptyList(), null, + FileSource.APPEND, Collections.emptyList(), null, null, + Collections.emptyList()); + DataSplit dataSplit = Mockito.mock(DataSplit.class); + Mockito.when(dataSplit.rowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.mergedRowCountAvailable()).thenReturn(true); + Mockito.when(dataSplit.mergedRowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.partition()).thenReturn(BinaryRow.singleColumn(1)); + Mockito.when(dataSplit.dataFiles()).thenReturn(Collections.singletonList(dataFileMeta)); + Mockito.when(dataSplit.convertToRawFiles()).thenReturn(Optional.empty()); + Mockito.when(dataSplit.deletionFiles()).thenReturn(Optional.empty()); + return dataSplit; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index 8cf98daea94c59..8ea9041480dadd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -17,14 +17,20 @@ package org.apache.doris.datasource.tvf.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.FunctionGenTable; +import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.datasource.FileSplitter; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.spi.Split; import org.apache.doris.tablefunction.ExternalFileTableValuedFunction; import org.apache.doris.thrift.TBrokerFileStatus; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TPushAggOp; import org.junit.Assert; import org.junit.Test; @@ -37,6 +43,40 @@ public class TVFScanNodeTest { private static final long MB = 1024L * 1024L; + @Test + public void testCountColumnKeepsNormalFileSplitting() throws Exception { + SessionVariable sv = new SessionVariable(); + sv.parallelExecInstanceNum = 1; + sv.setFileSplitSize(32 * MB); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_LOCAL); + Mockito.when(tvf.getFileStatuses()).thenReturn(List.of( + splittableFile("file:///tmp/count_col_1.parquet", 128 * MB), + splittableFile("file:///tmp/count_col_2.parquet", 128 * MB))); + desc.setTable(table); + + TVFScanNode countColumnNode = new TVFScanNode( + new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countColumnNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + + List countColumnSplits = countColumnNode.getSplits(1); + Assert.assertEquals(8, countColumnSplits.size()); + + TVFScanNode countStarNode = new TVFScanNode( + new PlanNodeId(1), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countStarNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + + List countStarSplits = countStarNode.getSplits(1); + Assert.assertEquals(2, countStarSplits.size()); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); @@ -57,4 +97,19 @@ public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Excep long target = (long) method.invoke(node, statuses); Assert.assertEquals(100 * MB, target); } + + private static TBrokerFileStatus splittableFile(String path, long size) { + TBrokerFileStatus status = new TBrokerFileStatus(); + status.setPath(path); + status.setSize(size); + status.setModificationTime(0); + status.setIsSplitable(true); + return status; + } + + private static void setFileSplitter(TVFScanNode node, FileSplitter splitter) throws Exception { + java.lang.reflect.Field field = FileQueryScanNode.class.getDeclaredField("fileSplitter"); + field.setAccessible(true); + field.set(node, splitter); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java index bf5b74f2ea6f25..eb3a42c3868610 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java @@ -75,7 +75,26 @@ public void testWithoutProject() { .applyImplementation(storageLayerAggregateWithoutProject()) .matches( logicalAggregate( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) + ) + ); + + // COUNT(*) still keeps a placeholder scan slot after column pruning, so its semantic + // argument list must remain empty instead of being inferred from the physical scan shape. + aggregate = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star")), + true, Optional.empty(), olapScan); + context = MemoTestUtils.createCascadesContext(aggregate); + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProject()) + .matches( + logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().isEmpty()) ) ); @@ -138,7 +157,9 @@ public void testWithProject() { .matches( logicalAggregate( logicalProject( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) ) ) ); diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 59c158e6d2d2ab..b943f246dfa9eb 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -1552,6 +1552,10 @@ struct TPlanNode { 50: optional list> distribute_expr_lists 51: optional bool is_serial_operator 52: optional TRecCTEScanNode rec_cte_scan_node + // COUNT(*) and COUNT(col) share push_down_agg_type_opt=COUNT, but file readers need to know + // whether a projected scan slot is the aggregate argument or merely the placeholder retained by + // column pruning. Empty means row-count semantics; non-empty identifies explicit COUNT columns. + 55: optional list push_down_count_slot_ids // projections is final projections, which means projecting into results and materializing them into the output block. 101: optional list projections diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv b/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..22eff58b7a23895a7e1d7f2e4323d6f5b1f9529d GIT binary patch literal 965 zcmaJ=-AWrl7@Zx5B}i|C&N2ggksB8a(Q2usl=b4td~q|Z`(h<;}#o1~a_NM_FWbIzRkVB6H)(?Z8OZfM0I)M0Ft6hask@E(b+rD$V= zQz)tcMIh!hbYqEJd&*fXQ(N}{A}g{BF`SxOA73G*I29DW6!b)-Y;4m+#9G?TP*}8U&PI39T~bT`I}sbR zT~-q5v(P3&u>}YzC$xnNgTiUu2hn|yd=FByVjBOP(O>efc=R)UZ`AAe=3akFHoF+l zqhYO5i+-#{Dyi`yY;2xTxBK?}8$UhsAfIEE!;FYJ{Kq;aDk&S%5z&yELzm0A;S8ur zbb^!P>f{K}cS4AcKe6a*2KjjlF_U=ZZiim(u6pOS;6B0{<|_(@`bwgfH*FDqHT;A@Xcye(6l_82M#A z+}gUjvn%K*)FEjaaq3|Om9Zx&-E8CcXoGn RZ5j*Vf8sZPfNlPO{{Z@npC14K literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5a9ca0baa47e892df900ff12a74417c3db63f91f GIT binary patch literal 923 zcmaJ=%TC)s6ulls7DC;y7;8rIA{(M00fnkmRTUQ3!3_vOlT=kV-8gBKra%+I!xBrr zpr6-|sQLvgxM!Ti&>%+ko%=fHKBQ@JXUC9y(BWC+9PXqRa{Db z9UzFbMQ}~LRJLJ)yP@b3WN4Uqts)*~>22pH|Y?UD^Idq4W;nbBm_nzCy6 zopzO7>W$83t{^KyL6;)!Vpp$HY?NCI$syf{H@b~(tIPElCwH;5D@;^h4_(y{UlP?& zFv#XycY%T2_ioDDWva3Nl%)v1tcf+ z5`3vI^~Kw{$mYpS*T>u7l@8|kh6g1+zrcGC)MNgdBKD@vR`5_$5JO-XmKX tXOoNT{?%r0Jl@PElTlu|=}#_)qhFObZ@0F$wp~3YDn9xn53D86@&QX@oumK& literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5b973181a56148d379ff1ee20612491d1121b700 GIT binary patch literal 673 zcmaJ<%TB^T6dh_)S-6nJG;QpH4a5Z?H5wmbGbIqzs90l2*qB-x#6YR#l@J%kFZ46~ z06)NUr_nYcaTa&(z2}}YXFBZ)*XE2{+^TRDiN-a{P!-0Qq2>^Ewx6jA1+)lMa|BYh zN&eg>^4C^EPOA)=KmxdGHjbI0&HU$J;&GL6rKA|jXpkhki*UAxqG>!229NX6BpAOH zEDMK1S~V@RQf3xc%#y}TB5`!}wZ1Or8~gQ$ z)4y}NTFzrb*VqxRd)qrtZfCqJ2eeLx0_%N z?~aO6NAl6fE YG%Ow-)#|mnS;70#6Yt@9y7)PN0MU(vxc~qF literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out new file mode 100644 index 00000000000000..c8312571c56e36 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !empty_csv -- + +-- !struct_leaf_promotion_cold -- +1 10 100 +3 10 300 + +-- !struct_leaf_promotion_warm -- +1 10 100 +3 10 300 + +-- !count_evolved_struct -- +4 + +-- !unprojected_unsupported_time -- +1 +2 diff --git a/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy new file mode 100644 index 00000000000000..6fa29c321c8e72 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_scanner_v2_review_fixes", "p0,external") { + def backends = sql "show backends" + assertTrue(backends.size() > 0) + def backendId = backends[0][0] + def dataPath = context.config.dataPath + "/external_table_p0/tvf" + // A developer may point this at fixtures already visible to every BE (for example, a shared + // checkout mounted at /tmp). CI leaves it unset and exercises the normal per-BE distribution. + def predeployedFixturePath = context.config.otherConfigs.get("fileScannerV2FixturePath") + def remotePath = predeployedFixturePath ?: "/tmp/file_scanner_v2_review_fixes" + def fixtureNames = [ + "file_scanner_v2_empty.csv", + "file_scanner_v2_struct_0_bigint.parquet", + "file_scanner_v2_struct_1_int.parquet", + "file_scanner_v2_unsupported_time.parquet" + ] + + if (predeployedFixturePath == null) { + mkdirRemotePathOnAllBE("root", remotePath) + for (def backend : backends) { + for (def fixtureName : fixtureNames) { + scpFiles("root", backend[1], "${dataPath}/${fixtureName}", remotePath, false) + } + } + } + + sql "set enable_file_scanner_v2 = true" + + // A zero-byte CSV is a valid zero-row split when the schema is supplied explicitly. The EOF + // raised during reader preparation must skip this split instead of failing the query. + order_qt_empty_csv """ + SELECT id, name + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_empty.csv", + "backend_id" = "${backendId}", + "format" = "csv", + "csv_schema" = "id:int;name:string") + ORDER BY id + """ + + // The first file defines STRUCT, while the second stores a as INT. Localization is + // split-specific: the BIGINT file compares BIGINT values, while the old INT file safely rewrites + // the exactly representable BIGINT literal 10 to INT and compares in the physical file type. + // If a literal cannot round-trip through INT, the mapper instead casts the INT data to BIGINT. + // Repeating the multi-file read also verifies that neither predicate rewrites nor page-cache + // range state leak from one file schema into the next file or the warm scan. + def evolvedStructQuery = """ + SELECT id, col.a, col.b + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE col.a = CAST(10 AS BIGINT) + ORDER BY id + """ + order_qt_struct_leaf_promotion_cold evolvedStructQuery + order_qt_struct_leaf_promotion_warm evolvedStructQuery + + // COUNT(col) sees a non-trivial mapping for the file whose STRUCT leaf is INT while the merged + // table type is BIGINT. It must fall back to the normal scan so schema-evolution casts and + // nullability checks run instead of counting footer values directly. + order_qt_count_evolved_struct """ + SELECT COUNT(col.a) + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + """ + + // TIME_MILLIS is intentionally unsupported by the record reader. It must not prevent schema + // construction when only the supported id column is projected. + order_qt_unprojected_unsupported_time """ + SELECT id + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + ORDER BY id + """ + + // A requested unsupported leaf must fail before metadata pruning. The fixture stores positive + // TIME_MILLIS values, so `unsupported_time < 0` can be eliminated from INT32 row-group + // statistics. Without the early validation, this query incorrectly returns an empty result + // instead of preserving the unsupported logical-type contract. + test { + sql """ + SELECT unsupported_time + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE unsupported_time < 0 + """ + exception "Parquet TIME with isAdjustedToUTC=true is not supported" + } + +} From 5e1b8d2a47ab5400dcd547850fc43a114218287a Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 20 Jul 2026 11:46:25 +0800 Subject: [PATCH 11/24] [fix](iceberg) make iceberg deletion vector stable. (#65676) Related PR: #59272 Doris already supports Iceberg v3 deletion vectors, but several stability gaps remain: - Invalid Iceberg/Paimon offsets and lengths could reach cache lookup or memory allocation. - Iceberg Puffin deletion-vector CRC32 was not verified. - Validation was inconsistent across legacy readers, format-v2 readers, normal scans, and the `position_deletes` path. - The Iceberg reader enforced a 1 GiB limit, while the writer did not. - Validate Iceberg/Paimon deletion-vector descriptors before cache lookup and allocation. - Verify Iceberg Puffin CRC32 before bitmap decoding. - Report malformed Paimon framing as data-quality errors. - Validate Puffin metadata in normal FE scans and the `position_deletes` path. - Align the Iceberg writer/reader 1 GiB limit and separate Iceberg/Paimon limits. - Add BE, FE, and format-v2 boundary tests. Harden Iceberg and Paimon deletion-vector validation and consistently enforce the 1 GiB Iceberg deletion-vector limit on write and read paths. - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. - Does this need documentation? - [x] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/exec/sink/viceberg_delete_sink.cpp | 20 +- be/src/exec/sink/viceberg_delete_sink.h | 4 + be/src/format/table/deletion_vector.h | 22 + .../format/table/deletion_vector_reader.cpp | 146 ++++++- be/src/format/table/deletion_vector_reader.h | 64 ++- .../iceberg_delete_file_reader_helper.cpp | 128 +++--- .../table/iceberg_delete_file_reader_helper.h | 15 +- be/src/format/table/paimon_reader.cpp | 70 ++-- be/src/format/table/paimon_reader.h | 8 + be/src/format_v2/deletion_vector.h | 12 + be/src/format_v2/table/iceberg_reader.cpp | 11 +- be/src/format_v2/table/paimon_reader.cpp | 6 +- .../exec/sink/viceberg_delete_sink_test.cpp | 14 + ...iceberg_delete_file_reader_helper_test.cpp | 381 +++++++++++++++++- .../format_v2/table/iceberg_reader_test.cpp | 25 +- .../format_v2/table/paimon_reader_test.cpp | 14 + .../source/IcebergDeleteFileFilter.java | 32 +- .../iceberg/source/IcebergScanNode.java | 8 +- .../source/IcebergDeleteFileFilterTest.java | 48 +++ .../iceberg/source/IcebergScanNodeTest.java | 30 ++ 20 files changed, 927 insertions(+), 131 deletions(-) create mode 100644 be/src/format/table/deletion_vector.h create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index e2b142db8d9f86..ef167ecdbcaea3 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -35,6 +35,7 @@ #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" #include "exprs/vexpr.h" +#include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/transformer/vfile_format_transformer.h" #include "io/file_factory.h" @@ -106,6 +107,20 @@ Status load_rewritable_delete_rows(RuntimeState* state, RuntimeProfile* profile, } // namespace +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size) { + DORIS_CHECK(content_size != nullptr); + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + if (bitmap_size > max_bitmap_size) { + return Status::NotSupported( + "Iceberg deletion vector bitmap size exceeds Doris supported limit: {}, " + "maximum bitmap size: {}, content size limit: {}", + bitmap_size, max_bitmap_size, MAX_ICEBERG_DELETION_VECTOR_BYTES); + } + *content_size = static_cast(bitmap_size + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); + return Status::OK(); +} + VIcebergDeleteSink::VIcebergDeleteSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs, std::shared_ptr dep, @@ -592,7 +607,8 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( blob.partition_spec_id = deletion.partition_spec_id; blob.partition_data_json = deletion.partition_data_json; blob.merged_count = static_cast(merged_rows.cardinality()); - blob.content_size_in_bytes = static_cast(4 + 4 + bitmap_size + 4); + RETURN_IF_ERROR(calculate_iceberg_deletion_vector_content_size( + bitmap_size, &blob.content_size_in_bytes)); blob.blob_data.resize(static_cast(blob.content_size_in_bytes)); merged_rows.write(blob.blob_data.data() + 8); @@ -604,7 +620,7 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( uint32_t crc = static_cast( ::crc32(0, reinterpret_cast(blob.blob_data.data() + 4), - 4 + (uInt)bitmap_size)); + static_cast(4 + bitmap_size))); BigEndian::Store32(blob.blob_data.data() + 8 + bitmap_size, crc); blobs.emplace_back(std::move(blob)); } diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 22ae98cc288100..647ee92f525733 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include #include @@ -41,6 +43,8 @@ class Dependency; class VIcebergDeleteFileWriter; +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size); + struct IcebergFileDeletion { IcebergFileDeletion() = default; IcebergFileDeletion(int32_t spec_id, std::string data_json) diff --git a/be/src/format/table/deletion_vector.h b/be/src/format/table/deletion_vector.h new file mode 100644 index 00000000000000..a4a029c2d6de9b --- /dev/null +++ b/be/src/format/table/deletion_vector.h @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// branch-4.1 still has both table-reader generations, so they must share one deletion-vector +// definition to avoid diverging limits and incompatible cache value types. +#include "format_v2/deletion_vector.h" diff --git a/be/src/format/table/deletion_vector_reader.cpp b/be/src/format/table/deletion_vector_reader.cpp index bfe34a5f555f94..8e8404839be9a8 100644 --- a/be/src/format/table/deletion_vector_reader.cpp +++ b/be/src/format/table/deletion_vector_reader.cpp @@ -17,29 +17,145 @@ #include "format/table/deletion_vector_reader.h" +#include + #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "util/block_compression.h" namespace doris { +namespace { + +constexpr int64_t ICEBERG_DELETION_VECTOR_MIN_BYTES = + static_cast(ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); +constexpr int64_t PAIMON_DELETION_VECTOR_MIN_BYTES = 8; +constexpr int64_t PAIMON_LENGTH_PREFIX_BYTES = 4; + +enum class DeletionVectorSizeLimitStatus { + DATA_QUALITY, + NOT_SUPPORTED, +}; + +Status validate_deletion_vector_read_range(const char* description, int64_t offset, int64_t size, + int64_t min_size, int64_t max_size, + DeletionVectorSizeLimitStatus size_limit_status, + size_t& bytes_read) { + if (offset < 0) { + return Status::DataQualityError("{} offset must be non-negative: {}", description, offset); + } + if (size < min_size) { + return Status::DataQualityError("{} size too small: {}, minimum: {}", description, size, + min_size); + } + if (size > max_size) { + if (size_limit_status == DeletionVectorSizeLimitStatus::NOT_SUPPORTED) { + return Status::NotSupported("{} size exceeds Doris supported limit: {}, limit: {}", + description, size, max_size); + } + return Status::DataQualityError("{} size exceeds limit: {}, limit: {}", description, size, + max_size); + } + if (offset > std::numeric_limits::max() - size) { + return Status::DataQualityError("{} offset plus size overflows: offset {}, size {}", + description, offset, size); + } + bytes_read = static_cast(size); + return Status::OK(); +} + +} // namespace + +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read) { + return validate_deletion_vector_read_range( + "Iceberg deletion vector", offset, size, ICEBERG_DELETION_VECTOR_MIN_BYTES, + MAX_ICEBERG_DELETION_VECTOR_BYTES, DeletionVectorSizeLimitStatus::NOT_SUPPORTED, + bytes_read); +} + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read) { + if (length < 0) { + return Status::DataQualityError("Paimon deletion vector length must be non-negative: {}", + length); + } + if (length > std::numeric_limits::max() - PAIMON_LENGTH_PREFIX_BYTES) { + return Status::DataQualityError("Paimon deletion vector length overflows: {}", length); + } + return validate_deletion_vector_read_range( + "Paimon deletion vector", offset, length + PAIMON_LENGTH_PREFIX_BYTES, + PAIMON_DELETION_VECTOR_MIN_BYTES, MAX_PAIMON_DELETION_VECTOR_BYTES, + DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read); +} + +DeletionVectorReader::~DeletionVectorReader() { + // The file reader may retain the child IOContext. Destroy it before merging and before the + // child statistics storage goes away. + _file_reader.reset(); + _merge_io_statistics(); +} + +void DeletionVectorReader::_init_io_context() { + if (_parent_io_ctx == nullptr) { + return; + } + _reader_io_ctx = *_parent_io_ctx; + _reader_io_ctx.file_cache_stats = &_file_cache_stats; + _reader_io_ctx.file_reader_stats = &_file_reader_stats; + _io_ctx = &_reader_io_ctx; +} + +void DeletionVectorReader::_merge_io_statistics() { + if (_statistics_merged || _parent_io_ctx == nullptr) { + return; + } + if (_parent_io_ctx->file_cache_stats != nullptr) { + _parent_io_ctx->file_cache_stats->merge_from(_file_cache_stats); + } + if (_parent_io_ctx->file_reader_stats != nullptr) { + _parent_io_ctx->file_reader_stats->read_calls += _file_reader_stats.read_calls; + _parent_io_ctx->file_reader_stats->read_bytes += _file_reader_stats.read_bytes; + _parent_io_ctx->file_reader_stats->read_time_ns += _file_reader_stats.read_time_ns; + _parent_io_ctx->file_reader_stats->read_rows += _file_reader_stats.read_rows; + } + _statistics_merged = true; +} + Status DeletionVectorReader::open() { if (_is_opened) [[unlikely]] { return Status::OK(); } + if (_desc.start_offset < 0 || _desc.size < 0) { + return Status::DataQualityError( + "Deletion vector range must be non-negative: path {}, offset {}, size {}", + _desc.path, _desc.start_offset, _desc.size); + } + _init_system_properties(); _init_file_description(); RETURN_IF_ERROR(_create_file_reader()); - _file_size = _file_reader->size(); + const size_t file_size = _file_reader->size(); + const size_t start_offset = static_cast(_desc.start_offset); + const size_t range_size = static_cast(_desc.size); + if (start_offset > file_size || range_size > file_size - start_offset) { + return Status::DataQualityError( + "Deletion vector range exceeds file size: path {}, offset {}, size {}, file size " + "{}", + _desc.path, _desc.start_offset, _desc.size, file_size); + } _is_opened = true; return Status::OK(); } Status DeletionVectorReader::read_at(size_t offset, Slice result) { - if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + if (UNLIKELY(_parent_io_ctx && _parent_io_ctx->should_stop)) { return Status::EndOfFile("stop read."); } + if (_io_ctx != nullptr) { + _io_ctx->should_stop = _parent_io_ctx->should_stop; + } size_t bytes_read = 0; RETURN_IF_ERROR(_file_reader->read_at(offset, result, &bytes_read, _io_ctx)); if (bytes_read != result.size) [[unlikely]] { @@ -50,13 +166,16 @@ Status DeletionVectorReader::read_at(size_t offset, Slice result) { } Status DeletionVectorReader::_create_file_reader() { - if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + if (UNLIKELY(_parent_io_ctx && _parent_io_ctx->should_stop)) { return Status::EndOfFile("stop read."); } - _file_description.mtime = _range.__isset.modification_time ? _range.modification_time : 0; + _file_description.mtime = _desc.modification_time; + // Keep DV blobs on the normal remote-file path. When file cache is enabled this creates a + // CachedRemoteFileReader, so a Puffin/delete-file range is persisted in the disk File Cache; + // the query-local decoded cache above it only owns the Roaring bitmap. io::FileReaderOptions reader_options = - FileFactory::get_reader_options(_state, _file_description); + FileFactory::get_reader_options(_state->query_options(), _file_description); _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( _profile, _system_properties, _file_description, reader_options, io::DelegateReader::AccessMode::RANDOM, _io_ctx)); @@ -64,20 +183,13 @@ Status DeletionVectorReader::_create_file_reader() { } void DeletionVectorReader::_init_file_description() { - _file_description.path = _range.path; - _file_description.file_size = _range.__isset.file_size ? _range.file_size : -1; - if (_range.__isset.fs_name) { - _file_description.fs_name = _range.fs_name; - } + _file_description.path = _desc.path; + _file_description.file_size = _desc.file_size; + _file_description.fs_name = _desc.fs_name; } void DeletionVectorReader::_init_system_properties() { - if (_range.__isset.file_type) { - // for compatibility - _system_properties.system_type = _range.file_type; - } else { - _system_properties.system_type = _params.file_type; - } + _system_properties.system_type = _params.file_type; _system_properties.properties = _params.properties; _system_properties.hdfs_params = _params.hdfs_params; if (_params.__isset.broker_addresses) { @@ -86,4 +198,4 @@ void DeletionVectorReader::_init_system_properties() { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/format/table/deletion_vector_reader.h b/be/src/format/table/deletion_vector_reader.h index 0663f3b28490ef..c187e2871687fe 100644 --- a/be/src/format/table/deletion_vector_reader.h +++ b/be/src/format/table/deletion_vector_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -24,10 +25,11 @@ #include "common/status.h" #include "format/generic_reader.h" +#include "format/table/deletion_vector.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" -#include "roaring/roaring64map.hh" +#include "io/io_common.h" #include "util/profile_collector.h" #include "util/slice.h" @@ -36,6 +38,28 @@ struct IOContext; } // namespace io namespace doris { +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read); + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read); + +struct DeleteFileDesc { + enum class Format { + PAIMON, + ICEBERG, + }; + + std::string key = ""; + std::string path = ""; + std::string fs_name = ""; + int64_t start_offset = 0; + int64_t size = 0; + int64_t file_size = -1; + int64_t modification_time = 0; + Format format = Format::PAIMON; +}; + class DeletionVectorReader { ENABLE_FACTORY_CREATOR(DeletionVectorReader); @@ -43,27 +67,57 @@ class DeletionVectorReader { DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, io::IOContext* io_ctx) - : _state(state), _profile(profile), _range(range), _params(params), _io_ctx(io_ctx) {} - ~DeletionVectorReader() = default; + : _state(state), _profile(profile), _params(params), _parent_io_ctx(io_ctx) { + _desc = DeleteFileDesc { + .key = "", + .path = range.path, + .fs_name = range.__isset.fs_name ? range.fs_name : "", + .start_offset = range.start_offset, + .size = range.size, + .file_size = range.__isset.file_size ? range.file_size : -1, + .modification_time = range.__isset.modification_time ? range.modification_time : 0}; + _init_io_context(); + } + DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, + const TFileScanRangeParams& params, const DeleteFileDesc& desc, + io::IOContext* io_ctx) + : _state(state), _profile(profile), _params(params), _parent_io_ctx(io_ctx) { + _desc = desc; + _init_io_context(); + } + ~DeletionVectorReader(); + DeletionVectorReader(const DeletionVectorReader&) = delete; + DeletionVectorReader& operator=(const DeletionVectorReader&) = delete; Status open(); Status read_at(size_t offset, Slice result); + // These are deliberately exposed separately from the decoded-DV cache counters. Local I/O is + // a hit in the disk-backed File Cache; remote I/O is a File Cache miss. A decoded-DV cache hit + // does not create a DeletionVectorReader and therefore changes neither value. + const io::FileCacheStatistics& file_cache_statistics() const { return _file_cache_stats; } + private: void _init_system_properties(); void _init_file_description(); + void _init_io_context(); + void _merge_io_statistics(); Status _create_file_reader(); private: RuntimeState* _state = nullptr; RuntimeProfile* _profile = nullptr; - const TFileRangeDesc& _range; + DeleteFileDesc _desc; const TFileScanRangeParams& _params; + io::IOContext* _parent_io_ctx = nullptr; + io::IOContext _reader_io_ctx; io::IOContext* _io_ctx = nullptr; + io::FileCacheStatistics _file_cache_stats; + io::FileReaderStats _file_reader_stats; + bool _statistics_merged = false; io::FileSystemProperties _system_properties; io::FileDescription _file_description; io::FileReaderSPtr _file_reader; - int64_t _file_size = 0; bool _is_opened = false; }; } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 0a2e79f1022f20..a193c552447a34 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -17,7 +17,7 @@ #include "format/table/iceberg_delete_file_reader_helper.h" -#include +#include #include #include @@ -36,17 +36,16 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "exec/common/endian.h" -#include "exprs/vexpr_context.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" #include "format/table/deletion_vector_reader.h" -#include "format/table/iceberg_reader.h" #include "format/table/table_format_reader.h" #include "io/hdfs_builder.h" -#include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "storage/predicate/column_predicate.h" +#include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris { @@ -54,6 +53,7 @@ namespace { constexpr const char* ICEBERG_FILE_PATH = "file_path"; constexpr const char* ICEBERG_ROW_POS = "pos"; +constexpr size_t ICEBERG_DELETION_VECTOR_MIN_BYTES = 12; const std::vector DELETE_COL_NAMES {ICEBERG_FILE_PATH, ICEBERG_ROW_POS}; std::unordered_map DELETE_COL_NAME_TO_BLOCK_IDX = {{ICEBERG_FILE_PATH, 0}, @@ -94,7 +94,6 @@ Status visit_position_delete_block(const Block& block, size_t read_rows, ColumnPtr path_column_ptr = block.get_by_position(path_it->second).column; ColumnPtr pos_column_ptr = block.get_by_position(pos_it->second).column; - // Iceberg permits optional schemas here, but a concrete delete key must never be null. if (const auto* nullable_col = check_and_get_column(*path_column_ptr); nullable_col != nullptr) { if (nullable_col->has_null(0, read_rows)) { @@ -136,32 +135,18 @@ Status visit_position_delete_block(const Block& block, size_t read_rows, return Status::InternalError("Unsupported file_path column type in position delete block"); } -Status init_parquet_delete_reader(ParquetReader* reader, bool* dictionary_coded) { - if (reader == nullptr || dictionary_coded == nullptr) { +Status init_parquet_delete_reader(ParquetReader* reader) { + if (reader == nullptr) { return Status::InvalidArgument("invalid parquet delete reader arguments"); } + VExprContextSPtrs conjuncts; phmap::flat_hash_map>> slot_id_to_predicates; - RETURN_IF_ERROR(reader->init_reader(DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, {}, + RETURN_IF_ERROR(reader->init_reader(DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, conjuncts, slot_id_to_predicates, nullptr, nullptr, nullptr, nullptr, nullptr, TableSchemaChangeHelper::ConstNode::get_instance(), false)); - std::unordered_map> - partition_columns; - std::unordered_map missing_columns; - RETURN_IF_ERROR(reader->set_fill_columns(partition_columns, missing_columns)); - - const tparquet::FileMetaData* meta_data = reader->get_meta_data(); - *dictionary_coded = true; - for (const auto& row_group : meta_data->row_groups) { - const auto& column_chunk = row_group.columns[0]; - if (!(column_chunk.__isset.meta_data && - IcebergTableReader::_is_fully_dictionary_encoded(column_chunk.meta_data))) { - *dictionary_coded = false; - break; - } - } return Status::OK(); } @@ -170,14 +155,10 @@ Status init_orc_delete_reader(OrcReader* reader) { return Status::InvalidArgument("orc delete reader is null"); } - RETURN_IF_ERROR(reader->init_reader(&DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, {}, false, - nullptr, nullptr, nullptr, nullptr, + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(reader->init_reader(&DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, conjuncts, + false, nullptr, nullptr, nullptr, nullptr, TableSchemaChangeHelper::ConstNode::get_instance())); - - std::unordered_map> - partition_columns; - std::unordered_map missing_columns; - RETURN_IF_ERROR(reader->set_fill_columns(partition_columns, missing_columns)); return Status::OK(); } @@ -186,14 +167,14 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, if (buf == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector decode arguments"); } - if (buffer_size < 12) { + if (buffer_size < ICEBERG_DELETION_VECTOR_MIN_BYTES) { return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); } - auto total_length = BigEndian::Load32(buf); - if (total_length + 8 != buffer_size) { + const uint32_t total_length = BigEndian::Load32(buf); + if (static_cast(total_length) + 8 != buffer_size) { return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", - total_length + 8, buffer_size); + static_cast(total_length) + 8, buffer_size); } constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; @@ -201,8 +182,17 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, return Status::DataQualityError("Deletion vector magic number mismatch"); } + const uint32_t expected_crc = BigEndian::Load32(buf + sizeof(total_length) + total_length); + const uint32_t actual_crc = + HashUtil::zlib_crc_hash(buf + sizeof(total_length), total_length, 0); + if (actual_crc != expected_crc) { + return Status::DataQualityError("Deletion vector CRC32 mismatch, expected: {}, actual: {}", + expected_crc, actual_crc); + } + try { - *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); + *rows_to_delete |= roaring::Roaring64Map::readSafe( + buf + 8, buffer_size - ICEBERG_DELETION_VECTOR_MIN_BYTES); } catch (const std::runtime_error& e) { return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); } @@ -215,7 +205,12 @@ IcebergDeleteFileIOContext::IcebergDeleteFileIOContext(RuntimeState* state) { io_ctx.file_cache_stats = &file_cache_stats; io_ctx.file_reader_stats = &file_reader_stats; if (state != nullptr) { + // branch-4.1 has no shared FileScanIOContext helper; keep delete-file reads attributed to + // the parent query and classified identically to ordinary external scans. io_ctx.query_id = &state->query_id(); + if (state->query_options().query_type == TQueryType::SELECT) { + io_ctx.reader_type = ReaderType::READER_QUERY; + } } } @@ -247,6 +242,25 @@ bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file) { return delete_file.__isset.content && delete_file.content == 3; } +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file) { + return fmt::format("delete_dv_{}:{}{}:{}#{}#{}", data_file_path.size(), data_file_path, + delete_file.path.size(), delete_file.path, delete_file.content_offset, + delete_file.content_size_in_bytes); +} + +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read) { + if (!delete_file.__isset.path || !delete_file.__isset.content_offset || + !delete_file.__isset.content_size_in_bytes) { + return Status::DataQualityError( + "Iceberg deletion vector descriptor misses " + "path/content_offset/content_size_in_bytes"); + } + return validate_iceberg_deletion_vector_read_range( + delete_file.content_offset, delete_file.content_size_in_bytes, bytes_read); +} + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor) { @@ -268,20 +282,15 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi options.batch_size, const_cast(&options.state->timezone_obj()), options.io_ctx, options.state, options.meta_cache); - bool dictionary_coded = false; - RETURN_IF_ERROR(init_parquet_delete_reader(&reader, &dictionary_coded)); + RETURN_IF_ERROR(init_parquet_delete_reader(&reader)); bool eof = false; while (!eof) { Block block; - if (dictionary_coded) { - block.insert(ColumnWithTypeAndName( - ColumnNullable::create(ColumnDictI32::create(), ColumnUInt8::create()), - make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); - } else { - block.insert(ColumnWithTypeAndName( - make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); - } + // branch-4.1 cannot safely nest ColumnDictI32 in ColumnNullable. Decode paths to + // strings so optional delete columns keep their null map for corruption checks. + block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), + ICEBERG_FILE_PATH)); block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), ICEBERG_ROW_POS)); size_t read_rows = 0; @@ -316,14 +325,17 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, - roaring::Roaring64Map* rows_to_delete) { + DeletionVector* rows_to_delete) { if (options.state == nullptr || options.profile == nullptr || options.scan_params == nullptr || options.io_ctx == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector reader options"); } - if (!delete_file.__isset.content_offset || !delete_file.__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(delete_file, bytes_read)); + DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.io_error", + { return Status::IOError("injected Iceberg deletion vector read failure"); }); + DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.should_stop", + { return Status::EndOfFile("stop read."); }); TFileRangeDesc delete_range = build_iceberg_delete_file_range(delete_file.path); if (options.fs_name != nullptr && !options.fs_name->empty()) { @@ -332,14 +344,24 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, delete_range.start_offset = delete_file.content_offset; delete_range.size = delete_file.content_size_in_bytes; + // Iceberg v3 deletion-vector-v1 blobs are uncompressed and metadata provides the exact range. + // Parse the Puffin footer first if future blob types or compression codecs are supported. DeletionVectorReader dv_reader(options.state, options.profile, *options.scan_params, delete_range, options.io_ctx); RETURN_IF_ERROR(dv_reader.open()); - std::vector buf(delete_range.size); - RETURN_IF_ERROR(dv_reader.read_at(delete_range.start_offset, - {buf.data(), cast_set(delete_range.size)})); - return decode_deletion_vector_buffer(buf.data(), delete_range.size, rows_to_delete); + std::vector buf(bytes_read); + const auto read_status = dv_reader.read_at(delete_range.start_offset, {buf.data(), bytes_read}); + if (options.deletion_vector_file_cache_stats != nullptr) { + options.deletion_vector_file_cache_stats->merge_from(dv_reader.file_cache_statistics()); + } + RETURN_IF_ERROR(read_status); + return decode_deletion_vector_buffer(buf.data(), bytes_read, rows_to_delete); +} + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete) { + return decode_deletion_vector_buffer(buf, buffer_size, rows_to_delete); } } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.h b/be/src/format/table/iceberg_delete_file_reader_helper.h index a6771212851ae8..adc0ef196f4b75 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.h +++ b/be/src/format/table/iceberg_delete_file_reader_helper.h @@ -26,6 +26,7 @@ #include #include "common/status.h" +#include "format/table/deletion_vector.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" @@ -50,6 +51,9 @@ struct IcebergDeleteFileReaderOptions { io::IOContext* io_ctx = nullptr; FileMetaCache* meta_cache = nullptr; const std::string* fs_name = nullptr; + // Optional per-DV attribution. The reader still merges these values into io_ctx so the + // scanner-wide File Cache profile remains complete. + io::FileCacheStatistics* deletion_vector_file_cache_stats = nullptr; size_t batch_size = 102400; }; @@ -67,12 +71,21 @@ TFileRangeDesc build_iceberg_delete_file_range(const std::string& path); bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file); +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file); + +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read); + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete); + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor); Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, - roaring::Roaring64Map* rows_to_delete); + DeletionVector* rows_to_delete); } // namespace doris diff --git a/be/src/format/table/paimon_reader.cpp b/be/src/format/table/paimon_reader.cpp index 0667ad8efff7f4..df0741b3e11920 100644 --- a/be/src/format/table/paimon_reader.cpp +++ b/be/src/format/table/paimon_reader.cpp @@ -17,14 +17,35 @@ #include "format/table/paimon_reader.h" +#include #include #include "common/status.h" +#include "exec/common/endian.h" #include "format/table/deletion_vector_reader.h" #include "runtime/runtime_state.h" namespace doris { #include "common/compile_check_begin.h" + +std::string build_paimon_deletion_vector_cache_key( + const TPaimonDeletionFileDesc& deletion_file) { + // A shared file can contain multiple vectors, so the cache identity must include its range. + return deletion_file.path + "#" + std::to_string(deletion_file.offset) + "#" + + std::to_string(deletion_file.length); +} + +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read) { + if (!deletion_file.__isset.path || !deletion_file.__isset.offset || + !deletion_file.__isset.length) { + return Status::DataQualityError( + "Paimon deletion file descriptor misses path/offset/length"); + } + return validate_paimon_deletion_vector_read_range(deletion_file.offset, deletion_file.length, + bytes_read); +} + PaimonReader::PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, const TFileRangeDesc& range, @@ -55,14 +76,11 @@ Status PaimonReader::init_row_filters() { _file_format_reader->set_push_down_agg_type(TPushAggOp::NONE); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); Status create_status = Status::OK(); - - std::string key; - key.resize(deletion_file.path.size() + sizeof(deletion_file.offset)); - memcpy(key.data(), deletion_file.path.data(), deletion_file.path.size()); - memcpy(key.data() + deletion_file.path.size(), &deletion_file.offset, - sizeof(deletion_file.offset)); + const std::string key = build_paimon_deletion_vector_cache_key(deletion_file); SCOPED_TIMER(_paimon_profile.delete_files_read_time); using DeleteRows = std::vector; @@ -74,7 +92,7 @@ Status PaimonReader::init_row_filters() { delete_range.__set_fs_name(_range.fs_name); delete_range.path = deletion_file.path; delete_range.start_offset = deletion_file.offset; - delete_range.size = deletion_file.length + 4; + delete_range.size = static_cast(bytes_read); delete_range.file_size = -1; DeletionVectorReader dv_reader(_state, _profile, _params, delete_range, _io_ctx); @@ -83,47 +101,31 @@ Status PaimonReader::init_row_filters() { return nullptr; } - // the reason of adding 4: https://github.com/apache/paimon/issues/3313 - size_t bytes_read = deletion_file.length + 4; - // TODO: better way to alloc memeory std::vector buffer(bytes_read); create_status = dv_reader.read_at(deletion_file.offset, {buffer.data(), bytes_read}); if (!create_status.ok()) [[unlikely]] { return nullptr; } - // parse deletion vector - const char* buf = buffer.data(); - uint32_t actual_length; - std::memcpy(reinterpret_cast(&actual_length), buf, 4); - // change byte order to big endian - std::reverse(reinterpret_cast(&actual_length), - reinterpret_cast(&actual_length) + 4); - buf += 4; - if (actual_length != bytes_read - 4) [[unlikely]] { - create_status = Status::RuntimeError( - "DeletionVector deserialize error: length not match, " - "actual length: {}, expect length: {}", - actual_length, bytes_read - 4); + const uint32_t actual_length = BigEndian::Load32(buffer.data()); + if (static_cast(actual_length) + 4 != bytes_read) [[unlikely]] { + create_status = Status::DataQualityError( + "Paimon deletion vector length mismatch, expected: {}, actual: {}", + static_cast(actual_length) + 4, bytes_read); return nullptr; } - uint32_t magic_number; - std::memcpy(reinterpret_cast(&magic_number), buf, 4); - // change byte order to big endian - std::reverse(reinterpret_cast(&magic_number), - reinterpret_cast(&magic_number) + 4); - buf += 4; - const static uint32_t MAGIC_NUMBER = 1581511376; - if (magic_number != MAGIC_NUMBER) [[unlikely]] { - create_status = Status::RuntimeError( - "DeletionVector deserialize error: invalid magic number {}", magic_number); + constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + if (memcmp(buffer.data() + 4, PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { + create_status = Status::DataQualityError( + "Paimon deletion vector magic number mismatch, expected: {}, actual: {}", + BigEndian::Load32(PAIMON_BITMAP_MAGIC), BigEndian::Load32(buffer.data() + 4)); return nullptr; } roaring::Roaring roaring_bitmap; SCOPED_TIMER(_paimon_profile.parse_deletion_vector_time); try { - roaring_bitmap = roaring::Roaring::readSafe(buf, bytes_read - 4); + roaring_bitmap = roaring::Roaring::readSafe(buffer.data() + 8, bytes_read - 8); } catch (const std::runtime_error& e) { create_status = Status::RuntimeError( "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", diff --git a/be/src/format/table/paimon_reader.h b/be/src/format/table/paimon_reader.h index de16c63cdd9a75..566841d947d093 100644 --- a/be/src/format/table/paimon_reader.h +++ b/be/src/format/table/paimon_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -26,6 +27,13 @@ namespace doris { #include "common/compile_check_begin.h" + +std::string build_paimon_deletion_vector_cache_key( + const TPaimonDeletionFileDesc& deletion_file); + +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read); + class PaimonReader : public TableFormatReader, public TableSchemaChangeHelper { public: PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, diff --git a/be/src/format_v2/deletion_vector.h b/be/src/format_v2/deletion_vector.h index 2c89771b2e6b0e..12ec7dbc6ebc80 100644 --- a/be/src/format_v2/deletion_vector.h +++ b/be/src/format_v2/deletion_vector.h @@ -17,10 +17,22 @@ #pragma once +#include +#include + #include "roaring/roaring64map.hh" namespace doris { +// Doris materializes an Iceberg deletion vector as one buffer. This is an implementation limit, +// not an Iceberg format limit. +inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30; +inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12; + +// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum serialized size is below this +// limit. Keep its guard independent from the Iceberg capability limit. +inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30; + // A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the // query-local cache instead of expanding every set bit into an int64_t. Position delete files use // a different representation because their input is a stream of (file_path, row_position) rows. diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 3791a1bb7a7436..fcbefcba1f0d81 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -387,18 +387,17 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes if (deletion_vector == nullptr) { return Status::OK(); } - if (!deletion_vector->__isset.content_offset || - !deletion_vector->__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(*deletion_vector, bytes_read)); const std::string data_file_path = iceberg_params.__isset.original_file_path ? iceberg_params.original_file_path : _data_file_path(); - desc->key = build_iceberg_deletion_vector_cache_key(data_file_path, *deletion_vector); + desc->key = ::doris::format::build_iceberg_deletion_vector_cache_key(data_file_path, + *deletion_vector); desc->path = deletion_vector->path; desc->start_offset = deletion_vector->content_offset; - desc->size = deletion_vector->content_size_in_bytes; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::ICEBERG; *has_delete_file = true; diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index f47d2a18fe7ac7..b22441c2b696e9 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -75,11 +75,13 @@ Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_d return Status::OK(); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); - desc->key = build_paimon_deletion_vector_cache_key(deletion_file); + desc->key = ::doris::format::build_paimon_deletion_vector_cache_key(deletion_file); desc->path = deletion_file.path; desc->start_offset = deletion_file.offset; - desc->size = deletion_file.length + 4; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::PAIMON; *has_delete_file = true; diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index d9fc5086503d90..7faa77ed702c68 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -33,6 +33,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "format/table/deletion_vector.h" #include "gen_cpp/DataSinks_types.h" #include "gen_cpp/Types_types.h" #include "runtime/runtime_state.h" @@ -507,6 +508,19 @@ TEST_F(VIcebergDeleteSinkTest, TestUnsupportedDeleteType) { ASSERT_FALSE(status.ok()); } +TEST_F(VIcebergDeleteSinkTest, TestValidateDeletionVectorContentSize) { + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + int64_t content_size = 0; + ASSERT_TRUE( + calculate_iceberg_deletion_vector_content_size(max_bitmap_size, &content_size).ok()); + ASSERT_EQ(MAX_ICEBERG_DELETION_VECTOR_BYTES, content_size); + + const auto unsupported_status = + calculate_iceberg_deletion_vector_content_size(max_bitmap_size + 1, &content_size); + ASSERT_TRUE(unsupported_status.is()) << unsupported_status; +} + TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { std::filesystem::path temp_dir = std::filesystem::temp_directory_path() / ("iceberg_delete_sink_test_" + generate_uuid_string()); diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp index 40d4a93639aea8..0b0ac1f6404e24 100644 --- a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp +++ b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp @@ -27,15 +27,22 @@ #include #include #include +#include #include #include #include #include #include +#include "exec/common/endian.h" +#include "format/table/deletion_vector_reader.h" #include "io/fs/file_meta_cache.h" +#include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/hash_util.hpp" namespace doris { @@ -47,6 +54,13 @@ constexpr const char* kMixedPositionDeleteFile = constexpr const char* kTargetDataFilePath = "s3://warehouse/wh/test_db/000_target_data_file.parquet"; +TUniqueId make_query_id() { + TUniqueId query_id; + query_id.hi = 300; + query_id.lo = 400; + return query_id; +} + class CollectPositionDeleteVisitor final : public IcebergPositionDeleteVisitor { public: Status visit(const std::string& file_path, int64_t pos) override { @@ -76,8 +90,8 @@ TIcebergDeleteFileDesc make_iceberg_deletion_vector(const std::string& path, int return delete_file; } -int64_t write_iceberg_deletion_vector_file(const std::string& file_path, - const std::vector& deleted_positions) { +std::vector build_iceberg_deletion_vector_blob( + const std::vector& deleted_positions) { roaring::Roaring64Map rows; for (const auto position : deleted_positions) { rows.add(position); @@ -91,7 +105,14 @@ int64_t write_iceberg_deletion_vector_file(const std::string& file_path, BigEndian::Store32(blob.data(), total_length); constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; memcpy(blob.data() + 4, DV_MAGIC, 4); - BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); + return blob; +} + +int64_t write_iceberg_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + const auto blob = build_iceberg_deletion_vector_blob(deleted_positions); std::ofstream output(file_path, std::ios::binary); EXPECT_TRUE(output.is_open()); @@ -136,7 +157,7 @@ void write_position_delete_parquet(const std::string& path, std::shared_ptr output; ASSERT_TRUE(arrow::io::FileOutputStream::Open(path).Value(&output).ok()); PARQUET_THROW_NOT_OK( - parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1024)); + ::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1024)); } IcebergDeleteFileReaderOptions make_delete_file_reader_options( @@ -187,9 +208,359 @@ TEST(IcebergDeleteFileReaderHelperTest, IsNotDeletionVectorWhenContentMissing) { EXPECT_FALSE(is_iceberg_deletion_vector(delete_file)); } +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorCacheKeyIncludesLocationAndRange) { + // Scenario: one Puffin file can hold several DV blobs. The cache key must isolate both the + // data file and the blob range so a scanner never reuses another file's DV. + TIcebergDeleteFileDesc first_delete_file; + first_delete_file.__set_path("s3://bucket/shared/delete.puffin"); + first_delete_file.__set_content_offset(128); + first_delete_file.__set_content_size_in_bytes(64); + + TIcebergDeleteFileDesc different_offset = first_delete_file; + different_offset.__set_content_offset(256); + + TIcebergDeleteFileDesc different_length = first_delete_file; + different_length.__set_content_size_in_bytes(96); + + TIcebergDeleteFileDesc different_delete_file = first_delete_file; + different_delete_file.__set_path("s3://bucket/shared/other-delete.puffin"); + + const std::string data_file_path = "s3://bucket/table/data-00001.parquet"; + const auto first_key = + build_iceberg_deletion_vector_cache_key(data_file_path, first_delete_file); + + EXPECT_NE(first_key, build_iceberg_deletion_vector_cache_key(data_file_path, different_offset)); + EXPECT_NE(first_key, build_iceberg_deletion_vector_cache_key(data_file_path, different_length)); + EXPECT_NE(first_key, + build_iceberg_deletion_vector_cache_key(data_file_path, different_delete_file)); + EXPECT_NE(first_key, + build_iceberg_deletion_vector_cache_key( + "s3://bucket/table/snapshot-branch/data-00001.parquet", first_delete_file)); +} + +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorCacheKeyEscapesPathBoundaries) { + TIcebergDeleteFileDesc first_delete_file; + first_delete_file.__set_path("middle#right#tail.puffin"); + first_delete_file.__set_content_offset(1); + first_delete_file.__set_content_size_in_bytes(2); + + TIcebergDeleteFileDesc second_delete_file = first_delete_file; + second_delete_file.__set_path("right#tail.puffin"); + + const std::string first_data_file_path = "s3://bucket/table/data#left"; + const std::string second_data_file_path = "s3://bucket/table/data#left#middle"; + ASSERT_EQ(first_data_file_path + "#" + first_delete_file.path, + second_data_file_path + "#" + second_delete_file.path); + + EXPECT_NE(build_iceberg_deletion_vector_cache_key(first_data_file_path, first_delete_file), + build_iceberg_deletion_vector_cache_key(second_data_file_path, second_delete_file)); +} + +TEST(IcebergDeleteFileReaderHelperTest, ValidateDeletionVectorDescriptor) { + size_t bytes_read = 0; + + TIcebergDeleteFileDesc missing_path; + missing_path.__set_content_offset(0); + missing_path.__set_content_size_in_bytes(12); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor(missing_path, bytes_read).ok()); + + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", -1, 12), bytes_read) + .ok()); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, 11), bytes_read) + .ok()); + EXPECT_TRUE( + validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES), + bytes_read) + .ok()); + EXPECT_EQ(static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES), bytes_read); + const auto unsupported_status = validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES + 1), + bytes_read); + EXPECT_TRUE(unsupported_status.is()) << unsupported_status; + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", + std::numeric_limits::max() - 10, 12), + bytes_read) + .ok()); + + EXPECT_TRUE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 7, 12), bytes_read) + .ok()); + EXPECT_EQ(bytes_read, 12); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsMissingFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_missing_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto missing_path = (test_dir / "missing-delete-vector.bin").string(); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector(make_iceberg_deletion_vector(missing_path, 0, 16), + options, &rows_to_delete); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(missing_path), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorRejectsRangePastFile) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_deletion_vector_range_past_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {1, 3}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector( + make_iceberg_deletion_vector(dv_path, 0, dv_size + 1), options, &rows_to_delete); + + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(status.to_string().find(dv_path), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorReaderValidatesOpenedFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_deletion_vector_opened_file_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {1, 3}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + + { + TFileRangeDesc exact_range = build_iceberg_delete_file_range(dv_path); + exact_range.start_offset = 4; + exact_range.size = dv_size - exact_range.start_offset; + DeletionVectorReader exact_reader(&state, &profile, scan_params, exact_range, + &io_context.io_ctx); + const auto exact_status = exact_reader.open(); + EXPECT_TRUE(exact_status.ok()) << exact_status; + + TFileRangeDesc oversized_range = exact_range; + oversized_range.size = MAX_ICEBERG_DELETION_VECTOR_BYTES; + DeletionVectorReader oversized_reader(&state, &profile, scan_params, oversized_range, + &io_context.io_ctx); + const auto oversized_status = oversized_reader.open(); + EXPECT_TRUE(oversized_status.is()) << oversized_status; + EXPECT_NE(oversized_status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(oversized_status.to_string().find(dv_path), std::string::npos); + } + + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorStopsWhenIoContextStops) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_stop_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + io_context.io_ctx.should_stop = true; + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector(make_iceberg_deletion_vector(dv_path, 0, dv_size), + options, &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("stop read"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCorruptPayload) { + std::vector corrupted(12, 0); + BigEndian::Store32(corrupted.data(), 4); + memcpy(corrupted.data() + 4, "BAD!", 4); + + roaring::Roaring64Map rows_to_delete; + auto status = decode_iceberg_deletion_vector_buffer(corrupted.data(), corrupted.size(), + &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); +} + +TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCrcMismatch) { + auto corrupted = build_iceberg_deletion_vector_blob({1, 3}); + corrupted.back() ^= 1; + + roaring::Roaring64Map rows_to_delete; + auto status = decode_iceberg_deletion_vector_buffer(corrupted.data(), corrupted.size(), + &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("CRC32 mismatch"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReadsMillionDeletePositions) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_large_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + constexpr uint64_t delete_position_count = 1'000'000; + std::vector deleted_positions; + deleted_positions.reserve(delete_position_count); + for (uint64_t position = 0; position < delete_position_count; ++position) { + deleted_positions.push_back(position * 2); + } + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, deleted_positions); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + ASSERT_TRUE(read_iceberg_deletion_vector(make_iceberg_deletion_vector(dv_path, 0, dv_size), + options, &rows_to_delete) + .ok()); + + EXPECT_EQ(rows_to_delete.cardinality(), delete_position_count); + EXPECT_TRUE(rows_to_delete.contains(static_cast(0))); + EXPECT_TRUE(rows_to_delete.contains(static_cast(999'998))); + EXPECT_TRUE(rows_to_delete.contains(static_cast(1'999'998))); + EXPECT_FALSE(rows_to_delete.contains(static_cast(1'999'999))); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorConcurrentReadsAreStable) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_deletion_vector_concurrent_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_dv_path = (test_dir / "delete-vector-first.bin").string(); + const auto second_dv_path = (test_dir / "delete-vector-second.bin").string(); + const auto first_dv_size = write_iceberg_deletion_vector_file(first_dv_path, {0, 2, 4}); + const auto second_dv_size = write_iceberg_deletion_vector_file(second_dv_path, {1, 3, 5}); + + auto read_and_verify = [](const std::string& path, int64_t size, + const std::vector& expected_positions) -> Status { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + RETURN_IF_ERROR(read_iceberg_deletion_vector(make_iceberg_deletion_vector(path, 0, size), + options, &rows_to_delete)); + if (rows_to_delete.cardinality() != expected_positions.size()) { + return Status::InternalError("unexpected deletion vector cardinality"); + } + for (const auto position : expected_positions) { + if (!rows_to_delete.contains(position)) { + return Status::InternalError("missing deletion vector position {}", position); + } + } + return Status::OK(); + }; + + std::vector workers; + std::vector statuses(16); + for (size_t idx = 0; idx < statuses.size(); ++idx) { + workers.emplace_back([&, idx]() { + SCOPED_INIT_THREAD_CONTEXT(); + if (idx % 2 == 0) { + statuses[idx] = read_and_verify(first_dv_path, first_dv_size, {0, 2, 4}); + } else { + statuses[idx] = read_and_verify(second_dv_path, second_dv_size, {1, 3, 5}); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + for (const auto& status : statuses) { + EXPECT_TRUE(status.ok()) << status; + } + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, IoContextPropagatesQueryIdentityForSelect) { + TQueryOptions query_options; + query_options.__set_query_type(TQueryType::SELECT); + + const auto query_id = make_query_id(); + MockRuntimeState state; + state._query_id = query_id; + state.set_query_options(query_options); + + IcebergDeleteFileIOContext io_context(&state); + + EXPECT_EQ(io_context.io_ctx.query_id, &state.query_id()); + EXPECT_EQ(io_context.io_ctx.reader_type, ReaderType::READER_QUERY); + EXPECT_EQ(io_context.io_ctx.file_cache_stats, &io_context.file_cache_stats); + EXPECT_EQ(io_context.io_ctx.file_reader_stats, &io_context.file_reader_stats); +} + +TEST(IcebergDeleteFileReaderHelperTest, IoContextDoesNotMarkLoadAsQueryReader) { + TQueryOptions query_options; + query_options.__set_query_type(TQueryType::LOAD); + + MockRuntimeState state; + state._query_id = make_query_id(); + state.set_query_options(query_options); + state._query_ctx = nullptr; + + IcebergDeleteFileIOContext io_context(&state); + + EXPECT_EQ(io_context.io_ctx.query_id, &state.query_id()); + EXPECT_EQ(io_context.io_ctx.reader_type, ReaderType::UNKNOWN); + EXPECT_EQ(io_context.io_ctx.file_cache_stats, &io_context.file_cache_stats); + EXPECT_EQ(io_context.io_ctx.file_reader_stats, &io_context.file_reader_stats); +} + TEST(IcebergDeleteFileReaderHelperTest, ReadMixedEncodingParquetPositionDeleteFile) { RuntimeProfile profile("test_profile"); - RuntimeState runtime_state((TQueryGlobals())); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); FileMetaCache meta_cache(1024); IcebergDeleteFileIOContext io_context(&runtime_state); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index f6bd784b6ecd08..6253ab853a8c3b 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -73,6 +73,7 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { @@ -675,7 +676,8 @@ int64_t write_iceberg_deletion_vector_file(const std::string& file_path, BigEndian::Store32(blob.data(), total_length); constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; memcpy(blob.data() + 4, DV_MAGIC, 4); - BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); std::ofstream output(file_path, std::ios::binary); EXPECT_TRUE(output.is_open()); @@ -1569,8 +1571,25 @@ TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsMissingRange) { auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("missing content offset or length"), std::string::npos); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("descriptor misses"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsInvalidRange) { + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_delete_files({make_iceberg_deletion_vector("dv.bin", -1, 12)}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); EXPECT_FALSE(has_delete_file); } diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 66e17ab4e687dc..08566f9253fcda 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -461,6 +461,20 @@ TEST(PaimonReaderTest, DeletionVectorCacheKeyIncludesOffsetAndLength) { EXPECT_NE(first_desc.key, different_length_desc.key); } +TEST(PaimonReaderTest, DeletionVectorRejectsInvalidRange) { + auto table_format_params = make_paimon_table_format_desc("dv.bin", -1, 4); + + paimon::PaimonReader reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = + reader.TEST_parse_deletion_vector_file(table_format_params, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + TEST(PaimonReaderTest, DecodeDeletionVectorBufferUsesSharedFormatHelper) { // Scenario: format_v2 TableReader reads a raw Paimon BitmapDeletionVector range and delegates // the binary parsing to the same helper used by the format reader path. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java index 32de4ebfdd9c0c..083409adda136c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java @@ -54,16 +54,46 @@ public static PositionDelete createPositionDelete(DeleteFile deleteFile) { String deleteFilePath = deleteFile.path().toString(); if (deleteFile.format() == FileFormat.PUFFIN) { + long fileSize = deleteFile.fileSizeInBytes(); + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + validateDeletionVectorMetadata(deleteFilePath, fileSize, contentOffset, contentLength); // The content_offset and content_size_in_bytes fields are used to reference // a specific blob for direct access to a deletion vector. return new DeletionVector(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), - deleteFile.fileSizeInBytes(), deleteFile.contentOffset(), deleteFile.contentSizeInBytes()); + fileSize, contentOffset, contentLength); } else { return new PositionDelete(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), deleteFile.fileSizeInBytes(), deleteFile.format()); } } + static void validateDeletionVectorMetadata( + String deleteFilePath, long fileSize, Long contentOffset, Long contentLength) { + if (contentOffset == null || contentLength == null) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata misses content offset or length: %s", deleteFilePath)); + } + if (fileSize < 0 || contentOffset < 0 || contentLength < 0) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata must be non-negative, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + if (contentOffset > Long.MAX_VALUE - contentLength) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range overflows, file: %s, content offset: %d, " + + "content length: %d", + deleteFilePath, contentOffset, contentLength)); + } + if (contentOffset + contentLength > fileSize) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range exceeds file size, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + } + public static EqualityDelete createEqualityDelete(String deleteFilePath, List fieldIds, long fileSize, FileFormat fileformat) { // todo: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 6bc485daab05fe..9ad496de40d622 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -1003,10 +1003,14 @@ private Split createIcebergPositionDeleteSysSplit(PositionDeletesScanTask task) split.setPositionDeleteFileFormat(getNativePositionDeleteFileFormat(deleteFile.format())); split.setPositionDeleteOriginalPath(originalPath); if (deleteFile.format() == FileFormat.PUFFIN) { + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + IcebergDeleteFileFilter.validateDeletionVectorMetadata( + originalPath, deleteFile.fileSizeInBytes(), contentOffset, contentLength); split.setPositionDeleteContent(IcebergDeleteFileFilter.DeletionVector.type()); split.setPositionDeleteReferencedDataFilePath(deleteFile.referencedDataFile()); - split.setPositionDeleteContentOffset(deleteFile.contentOffset()); - split.setPositionDeleteContentSizeInBytes(deleteFile.contentSizeInBytes()); + split.setPositionDeleteContentOffset(contentOffset); + split.setPositionDeleteContentSizeInBytes(contentLength); } else { split.setPositionDeleteContent(IcebergDeleteFileFilter.PositionDelete.type()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java new file mode 100644 index 00000000000000..6a714107a6e856 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.source; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IcebergDeleteFileFilterTest { + @Test + public void testValidateDeletionVectorMetadataAcceptsLongRange() { + Assertions.assertDoesNotThrow(() -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", 1L << 40, (1L << 32) + 17, (1L << 30) + 19)); + } + + @Test + public void testValidateDeletionVectorMetadataRejectsInvalidRange() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, null, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", -1, 1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, -1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, -1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", Long.MAX_VALUE, Long.MAX_VALUE, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 90L, 11L)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 88bdbdd6d1018f..98136848f56ca1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -32,10 +32,12 @@ import org.apache.doris.thrift.TPushAggOp; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -273,6 +275,34 @@ public void testSetIcebergParamsUsesSplitFileFormat() throws Exception { Assert.assertEquals(TFileFormatType.FORMAT_ORC, rangeDesc.getFormatType()); } + @Test + public void testPositionDeleteSystemTableValidatesDeletionVectorMetadata() throws Exception { + DeleteFile deleteFile = Mockito.mock(DeleteFile.class); + Mockito.when(deleteFile.path()).thenReturn("file:///tmp/delete-shared.puffin"); + Mockito.when(deleteFile.format()).thenReturn(FileFormat.PUFFIN); + Mockito.when(deleteFile.fileSizeInBytes()).thenReturn(100L); + Mockito.when(deleteFile.contentOffset()).thenReturn(null); + Mockito.when(deleteFile.contentSizeInBytes()).thenReturn(10L); + + PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); + Mockito.when(task.file()).thenReturn(deleteFile); + Mockito.when(task.start()).thenReturn(0L); + Mockito.when(task.length()).thenReturn(100L); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Method method = IcebergScanNode.class.getDeclaredMethod( + "createIcebergPositionDeleteSysSplit", PositionDeletesScanTask.class); + method.setAccessible(true); + + try { + method.invoke(node, task); + Assert.fail("position_deletes planning should reject invalid deletion vector metadata"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof IllegalArgumentException); + Assert.assertTrue(e.getCause().getMessage().contains("delete-shared.puffin")); + } + } + @Test public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exception { SessionVariable sv = new SessionVariable(); From cb671adc08dea6a1600325a8002fadbd87bd3297 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 21 Jul 2026 14:26:03 +0800 Subject: [PATCH 12/24] [fix](topn) Resolve topn lazy materialization column indexes for aliases (#65759) Related pr #52114 Problem Summary: Problem Summary: TopN lazy materialization resolved deferred column indexes with output slot names. Queries that renamed Hive columns therefore produced -1 indexes, and external row-id fetch could fill those columns with NULL when positional reading was used. Resolve the index from the already traced original base column so the descriptor and index share one column identity. Add a focused planner unit test and a Hive ORC Explain regression for the alias path. Fix incorrect NULL values from aliased external-table columns when TopN lazy materialization is used. --- .../physical/PhysicalLazyMaterialize.java | 2 +- .../postprocess/TopnLazyMaterializeTest.java | 59 +++++++++++++++++++ .../hive/test_hive_topn_lazy_mat.groovy | 9 +++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java index fad35834d90cf0..51a8bf054f5294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java @@ -170,7 +170,7 @@ public PhysicalLazyMaterialize(CHILD_TYPE child, Column originalColumn = materializeMap.get(lazySlot).baseSlot.getOriginalColumn().get(); outputBuilder.add(((SlotReference) lazySlot).withColumn(originalColumn)); lazyColumnForRel.add(originalColumn); - lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(lazySlot.getName())); + lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(originalColumn.getName())); lazySlotLocationForRel.add(loc); loc++; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index 159cd611676086..1e7d6b39154b11 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.processor.post.PlanPostProcessors; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.MaterializationNode; @@ -80,6 +82,63 @@ public void test2() throws Exception { Assertions.assertEquals("k2", slots.get(0).getColumn().getName()); } + @Test + public void testNestedColumnAccessPathInLazyMaterialize() throws Exception { + this.createTables("create table lazy_materialize_struct_tbl(" + + "id bigint, " + + "user_profile struct<" + + "personal:struct," + + "professional:struct>>) " + + "duplicate key(id) distributed by hash(id) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select struct_element(struct_element(user_profile, 'professional'), 'skills') " + + "from lazy_materialize_struct_tbl order by id limit 10"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + PlanTranslatorContext translatorContext = new PlanTranslatorContext(checker.getCascadesContext()); + PlanFragment fragment = new PhysicalPlanTranslator(translatorContext).translatePlan(plan); + + List materializationNodes = Lists.newArrayList(); + fragment.getPlanRoot().collect(MaterializationNode.class, materializationNodes); + Assertions.assertEquals(1, materializationNodes.size()); + TupleDescriptor materializeTupleDesc = translatorContext.getTupleDesc( + materializationNodes.get(0).getTupleIds().get(0)); + SlotDescriptor userProfileSlot = materializeTupleDesc.getSlots().stream() + .filter(slot -> "user_profile".equals(slot.getColumn().getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("lazy user_profile slot not found")); + Assertions.assertTrue(userProfileSlot.getAllAccessPaths().contains( + ColumnAccessPath.data(ImmutableList.of("user_profile", "professional", "skills")))); + } + + @Test + public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Exception { + this.createTable("create table lazy_materialize_alias_tbl(" + + "sort_col int, lazy_col int) " + + "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select lazy_col as lazy_alias from lazy_materialize_alias_tbl " + + "order by sort_col limit 1"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> materializeNodes = plan.collectToList( + node -> node instanceof PhysicalLazyMaterialize); + Assertions.assertEquals(1, materializeNodes.size(), plan.treeString()); + Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), + materializeNodes.get(0).getLazyBaseColumnIndices()); + } + @Test public void testLightSchemaChangeFalse() throws Exception { this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) " diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index 189634a4134704..222d3a393b215e 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -223,6 +223,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") } + // Output aliases must not be used to resolve physical Hive column indices. + explain { + sql "select name as lazy_alias from orc_topn_lazy_mat_table order by id limit 10;" + contains("VMaterializeNode") + contains("column_descs_lists[[`name` text NULL]]") + contains("column_idxs_lists: [[1]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") + } + explain { sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """ From c0a2f822af92c735fac14440f98255835c6f99ce Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 21 Jul 2026 16:07:56 +0800 Subject: [PATCH 13/24] [fix](iceberg) Honor disabled write metrics (#65782) - honor Iceberg table metrics configuration when Doris creates data-file metadata - omit all column metrics whose effective Iceberg metrics mode is `none` - omit bounds for `counts` and safely truncate string/binary bounds for `truncate(N)` - build the table metrics policy once per commit batch instead of once per output file - return unknown Iceberg column statistics when required file metrics are absent - preserve unknown statistics when closing an Iceberg file scan fails - add regression coverage for the write-to-statistics path Doris collected column statistics in the backend and copied every statistics map into the Iceberg `DataFile` manifest in the frontend. The conversion never consulted the table's `MetricsConfig`, so metadata for disabled columns was persisted even though the physical file statistics were available only as an implementation detail. The initial filtering also treated every non-`none` mode like `full`, retaining bounds for `counts` and failing to safely truncate string/binary bounds for `truncate(N)`. After disabled metrics were correctly omitted, the downstream Iceberg statistics reader still assumed `columnSizes` and `nullValueCounts` always contained every column. It could therefore throw while loading statistics for a table using metrics mode `none`. A scan-close failure could also override an in-flight unknown-statistics return and expose partial or fabricated accumulators. The reader now treats missing required metrics and scan-close failures as unknown. - `IcebergWriterHelperTest` and `StatisticsUtilTest` (21 tests passed) - verified the new review regression tests fail before their production fixes and pass after them - FE CheckStyle validation (0 violations) - `git diff --check` - Jira: http://39.106.86.136:8090/browse/DORIS-27023 - TeamCity reproduction: http://172.20.48.17:8111/buildConfiguration/Doris_Doris_x64_Master_Trino_Case/201908 --- be/src/exec/sink/viceberg_merge_sink.cpp | 3 + .../iceberg/viceberg_partition_writer.cpp | 10 +- .../iceberg/viceberg_partition_writer.h | 3 + .../format/transformer/vorc_transformer.cpp | 11 +- .../iceberg/iceberg_partition_writer_test.cpp | 107 ++++++++ .../transformer/vorc_transformer_test.cpp | 107 ++++++++ .../datasource/iceberg/IcebergUtils.java | 24 +- .../iceberg/helper/IcebergWriterHelper.java | 117 +++++++- .../doris/planner/IcebergMergeSink.java | 1 + .../doris/planner/IcebergTableSink.java | 1 + .../doris/statistics/util/StatisticsUtil.java | 14 +- .../helper/IcebergWriterHelperTest.java | 251 ++++++++++++++++++ .../doris/planner/IcebergMergeSinkTest.java | 70 ++++- .../statistics/util/StatisticsUtilTest.java | 70 +++++ gensrc/thrift/DataSinks.thrift | 4 + 15 files changed, 780 insertions(+), 13 deletions(-) create mode 100644 be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp create mode 100644 be/test/format/transformer/vorc_transformer_test.cpp diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp index 7d6c84cf068dbc..e9f4d6bf5c655a 100644 --- a/be/src/exec/sink/viceberg_merge_sink.cpp +++ b/be/src/exec/sink/viceberg_merge_sink.cpp @@ -254,6 +254,9 @@ Status VIcebergMergeSink::_build_inner_sinks() { if (merge_sink.__isset.broker_addresses) { table_sink.__set_broker_addresses(merge_sink.broker_addresses); } + if (merge_sink.__isset.collect_column_stats) { + table_sink.__set_collect_column_stats(merge_sink.collect_column_stats); + } _table_sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); _table_sink.__set_iceberg_table_sink(table_sink); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 434488266bb3ac..25da8724b2899b 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -47,7 +47,11 @@ VIcebergPartitionWriter::VIcebergPartitionWriter( _file_name_index(file_name_index), _file_format_type(file_format_type), _compress_type(compress_type), - _hadoop_conf(hadoop_conf) {} + _hadoop_conf(hadoop_conf) { + if (t_sink.iceberg_table_sink.__isset.collect_column_stats) { + _collect_column_stats = t_sink.iceberg_table_sink.collect_column_stats; + } +} Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile, const RowDescriptor* row_desc) { @@ -155,6 +159,10 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c commit_data->__set_file_size(_file_format_transformer->written_len()); commit_data->__set_file_content(TFileContent::DATA); commit_data->__set_partition_values(_partition_values); + // ORC collection reopens the file, so honor the FE policy before any footer work. + if (!_collect_column_stats) { + return Status::OK(); + } if (_file_format_type == TFileFormatType::FORMAT_PARQUET) { TIcebergColumnStats column_stats; RETURN_IF_ERROR(static_cast(_file_format_transformer.get()) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h index 97b4dd3efdfac0..b0839a82ed3796 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h @@ -67,6 +67,8 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { inline size_t written_len() const override { return _file_format_transformer->written_len(); } private: + friend class VIcebergPartitionWriterTest; + std::string _get_target_file_name(); Status _build_iceberg_commit_data(TIcebergCommitData* commit_data); @@ -91,6 +93,7 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { TFileFormatType::type _file_format_type; TFileCompressType::type _compress_type; const std::map& _hadoop_conf; + bool _collect_column_stats = true; std::shared_ptr _fs = nullptr; diff --git a/be/src/format/transformer/vorc_transformer.cpp b/be/src/format/transformer/vorc_transformer.cpp index 3e3949cd924469..db2ca74c64acff 100644 --- a/be/src/format/transformer/vorc_transformer.cpp +++ b/be/src/format/transformer/vorc_transformer.cpp @@ -386,12 +386,19 @@ Status VOrcTransformer::collect_file_statistics_after_close(TIcebergColumnStats* const iceberg::StructType& root_struct = _iceberg_schema->root_struct(); const auto& nested_fields = root_struct.fields(); + const orc::Type& orc_root_type = reader->getType(); for (uint32_t i = 0; i < nested_fields.size(); i++) { - uint32_t orc_col_id = i + 1; // skip root struct - if (orc_col_id >= file_stats->getNumberOfColumns()) { + if (i >= orc_root_type.getSubtypeCount()) { + continue; + } + // ORC IDs are depth-first, so top-level fields after a complex field are not i + 1. + const uint64_t raw_orc_col_id = orc_root_type.getSubtype(i)->getColumnId(); + if (raw_orc_col_id >= file_stats->getNumberOfColumns()) { continue; } + // The uint32_t column-count check above makes narrowing to the ORC API width safe. + const uint32_t orc_col_id = static_cast(raw_orc_col_id); const orc::ColumnStatistics* col_stats = file_stats->getColumnStatistics(orc_col_id); if (col_stats == nullptr) { continue; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp new file mode 100644 index 00000000000000..d453177cf25044 --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "exec/sink/writer/iceberg/viceberg_partition_writer.h" + +namespace doris { + +namespace { + +class FakeFileFormatTransformer final : public VFileFormatTransformer { +public: + explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs) + : VFileFormatTransformer(nullptr, output_exprs, false) {} + + Status open() override { return Status::OK(); } + Status write(const Block&) override { return Status::OK(); } + Status close() override { return Status::OK(); } + int64_t written_len() override { return 64; } +}; + +TDataSink make_table_sink(std::optional collect_column_stats) { + TIcebergTableSink iceberg_sink; + if (collect_column_stats.has_value()) { + iceberg_sink.__set_collect_column_stats(*collect_column_stats); + } + TDataSink sink; + sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); + sink.__set_iceberg_table_sink(iceberg_sink); + return sink; +} + +} // namespace + +class VIcebergPartitionWriterTest : public testing::Test { +protected: + static std::unique_ptr make_writer( + const TDataSink& sink, const VExprContextSPtrs& output_exprs, + const iceberg::Schema& schema, const std::string* schema_json, + const std::map& hadoop_conf) { + IPartitionWriterBase::WriteInfo write_info; + write_info.file_type = TFileType::FILE_LOCAL; + return std::make_unique( + sink, std::vector {}, output_exprs, schema, schema_json, + std::vector {}, std::move(write_info), "data", 0, + TFileFormatType::FORMAT_ORC, TFileCompressType::ZLIB, hadoop_conf); + } + + static void install_fake_transformer(VIcebergPartitionWriter* writer, + const VExprContextSPtrs& output_exprs) { + writer->_file_format_transformer = + std::make_unique(output_exprs); + } + + static Status build_commit_data(VIcebergPartitionWriter* writer, + TIcebergCommitData* commit_data) { + return writer->_build_iceberg_commit_data(commit_data); + } + + static bool collect_column_stats(const VIcebergPartitionWriter& writer) { + return writer._collect_column_stats; + } +}; + +TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabled) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf); + install_fake_transformer(writer.get(), output_exprs); + + TIcebergCommitData commit_data; + ASSERT_TRUE(build_commit_data(writer.get(), &commit_data).ok()); + EXPECT_FALSE(commit_data.__isset.column_stats); +} + +TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = make_writer(make_table_sink(std::nullopt), output_exprs, schema, &schema_json, + hadoop_conf); + + EXPECT_TRUE(collect_column_stats(*writer)); +} + +} // namespace doris diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp new file mode 100644 index 00000000000000..4ea14766356d15 --- /dev/null +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/transformer/vorc_transformer.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/table/iceberg/schema_parser.h" +#include "io/fs/local_file_system.h" +#include "runtime/runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" +#include "util/uid_util.h" + +namespace doris { + +class VOrcTransformerTest : public testing::Test { +protected: + void SetUp() override { + _file_path = "./vorc_transformer_" + UniqueId::gen_uid().to_string() + ".orc"; + _fs = io::global_local_filesystem(); + } + + void TearDown() override { static_cast(_fs->delete_file(_file_path)); } + + std::string _file_path; + std::shared_ptr _fs; +}; + +TEST_F(VOrcTransformerTest, CollectsBoundsForTopLevelFieldAfterStruct) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto string_type = std::make_shared(); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {struct_type, string_type}); + + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + { + "id": 1, + "name": "s", + "required": true, + "type": { + "type": "struct", + "fields": [ + {"id": 2, "name": "a", "required": true, "type": "int"} + ] + } + }, + {"id": 3, "name": "b", "required": true, "type": "string"} + ] + })"; + std::unique_ptr schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + VOrcTransformer transformer(&state, file_writer.get(), output_exprs, "", {"s", "b"}, false, + TFileCompressType::PLAIN, schema.get(), _fs); + ASSERT_TRUE(transformer.open().ok()); + + auto nested_column = ColumnInt32::create(); + nested_column->insert_value(-1); + Columns struct_columns; + struct_columns.emplace_back(std::move(nested_column)); + auto struct_column = ColumnStruct::create(std::move(struct_columns)); + auto string_column = ColumnString::create(); + string_column->insert_data("hello", 5); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(struct_column), struct_type, "s")); + block.insert(ColumnWithTypeAndName(std::move(string_column), string_type, "b")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + TIcebergColumnStats stats; + ASSERT_TRUE(transformer.collect_file_statistics_after_close(&stats).ok()); + ASSERT_TRUE(stats.__isset.lower_bounds); + ASSERT_TRUE(stats.__isset.upper_bounds); + ASSERT_EQ(1, stats.lower_bounds.count(3)); + ASSERT_EQ(1, stats.upper_bounds.count(3)); + EXPECT_EQ("hello", stats.lower_bounds.at(3)); + EXPECT_EQ("hello", stats.upper_bounds.at(3)); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index a78bdc57c7afb1..a7908db222cd00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -77,14 +77,17 @@ import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -1903,10 +1906,25 @@ public static Schema appendRowLineageFieldsForV3(Schema schema) { MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER)); } + public static boolean shouldCollectColumnStats(Table table, Schema writerSchema) { + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + if (getFileFormat(table) == FileFormat.ORC) { + // Match the footer collectors: ORC reports top-level collection counts, while Parquet reports leaf fields. + return writerSchema.columns().stream() + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + return TypeUtil.indexById(writerSchema.asStruct()).values().stream() + .filter(field -> field.type().isPrimitiveType()) + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + public static int getFormatVersion(Table table) { int formatVersion = 2; // default format version : 2 - if (table instanceof BaseTable) { - formatVersion = ((BaseTable) table).operations().current().formatVersion(); + if (table instanceof HasTableOperations) { + // TransactionTable exposes the real format version through operations, not table properties. + formatVersion = ((HasTableOperations) table).operations().current().formatVersion(); } else if (table != null && table.properties() != null) { String version = table.properties().get(TableProperties.FORMAT_VERSION); if (version != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java index b67a5911b64384..54a791e7e18133 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java @@ -30,12 +30,21 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.BinaryUtil; +import org.apache.iceberg.util.UnicodeUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,6 +70,12 @@ public static WriteResult convertToWriterResult( // Get table specification information PartitionSpec spec = table.spec(); FileFormat fileFormat = IcebergUtils.getFileFormat(table); + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + Schema schema = table.schema(); + if (IcebergUtils.getFormatVersion(table) >= IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) { + // Rewrite and merge writers emit v3 lineage columns that are absent from the table schema. + schema = IcebergUtils.appendRowLineageFieldsForV3(schema); + } for (TIcebergCommitData commitData : commitDataList) { //get the files path @@ -70,7 +85,7 @@ public static WriteResult convertToWriterResult( long fileSize = commitData.getFileSize(); long recordCount = commitData.getRowCount(); CommonStatistics stat = new CommonStatistics(recordCount, DEFAULT_FILE_COUNT, fileSize); - Metrics metrics = buildDataFileMetrics(table, fileFormat, commitData); + Metrics metrics = buildDataFileMetrics(commitData, schema, metricsConfig, fileFormat); Optional partitionData = Optional.empty(); //get and check partitionValues when table is partitionedTable if (spec.isPartitioned()) { @@ -153,7 +168,9 @@ private static PartitionData convertToPartitionData( return partitionData; } - private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, TIcebergCommitData commitData) { + private static Metrics buildDataFileMetrics( + TIcebergCommitData commitData, Schema schema, MetricsConfig metricsConfig, FileFormat fileFormat) { + Map fieldParents = TypeUtil.indexParents(schema.asStruct()); Map columnSizes = new HashMap<>(); Map valueCounts = new HashMap<>(); Map nullValueCounts = new HashMap<>(); @@ -178,8 +195,100 @@ private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, } } - return new Metrics(commitData.getRowCount(), columnSizes, valueCounts, - nullValueCounts, null, lowerBounds, upperBounds); + // Physical file stats may contain every column, but manifest metrics must honor the table's metadata policy. + return new Metrics(commitData.getRowCount(), + filterDisabledMetrics(columnSizes, schema, metricsConfig), + filterLogicalMetrics(valueCounts, schema, metricsConfig, fieldParents), + filterLogicalMetrics(nullValueCounts, schema, metricsConfig, fieldParents), + null, + filterBounds(lowerBounds, schema, metricsConfig, fieldParents, fileFormat, true), + filterBounds(upperBounds, schema, metricsConfig, fieldParents, fileFormat, false)); + } + + private static Map filterDisabledMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + if (MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterLogicalMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig, + Map fieldParents) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + // Definition-level values below list/map do not represent logical element counts. + if (!isInRepeatedField(fieldId, schema, fieldParents) + && MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterBounds( + Map bounds, Schema schema, MetricsConfig metricsConfig, + Map fieldParents, FileFormat fileFormat, boolean lowerBound) { + Map filteredBounds = new HashMap<>(); + bounds.forEach((fieldId, value) -> { + if (isInRepeatedField(fieldId, schema, fieldParents)) { + return; + } + MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema, metricsConfig, fieldId); + if (mode == MetricsModes.None.get() || mode == MetricsModes.Counts.get()) { + return; + } + + ByteBuffer filteredValue = value; + if (mode instanceof MetricsModes.Truncate) { + Type type = schema.findType(fieldId); + int length = ((MetricsModes.Truncate) mode).length(); + // Truncated upper bounds must round up so file pruning cannot exclude matching values. + filteredValue = truncateBound(type, value, length, fileFormat, lowerBound); + } + if (filteredValue != null) { + filteredBounds.put(fieldId, filteredValue); + } + }); + return filteredBounds; + } + + private static boolean isInRepeatedField( + int fieldId, Schema schema, Map fieldParents) { + Integer parentId = fieldId; + while ((parentId = fieldParents.get(parentId)) != null) { + Types.NestedField parent = schema.findField(parentId); + if (parent != null && !parent.type().isStructType()) { + return true; + } + } + return false; + } + + private static ByteBuffer truncateBound( + Type type, ByteBuffer value, int length, FileFormat fileFormat, boolean lowerBound) { + switch (type.typeId()) { + case STRING: + String stringValue = Conversions.fromByteBuffer(type, value).toString(); + String truncatedString = lowerBound + ? UnicodeUtil.truncateStringMin(stringValue, length) + : UnicodeUtil.truncateStringMax(stringValue, length); + // ORC keeps the full maximum when no safe truncated successor exists. + if (!lowerBound && truncatedString == null && fileFormat == FileFormat.ORC) { + return value; + } + return truncatedString == null ? null : Conversions.toByteBuffer(type, truncatedString); + case BINARY: + return lowerBound + ? BinaryUtil.truncateBinaryMin(value, length) + : BinaryUtil.truncateBinaryMax(value, length); + default: + return value; + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java index 4af4ba17e18578..9775f81c172fae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java @@ -131,6 +131,7 @@ public void bindDataSink(Optional insertCtx) } tSink.setFormatVersion(formatVersion); tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java index 0f3b1bb24d26bc..b7d3da47cb4d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -135,6 +135,7 @@ public void bindDataSink(Optional insertCtx) schema = IcebergUtils.appendRowLineageFieldsForV3(schema); } tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 6a0b7c71ffd057..c59992082ad54e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -730,12 +730,22 @@ public static Optional getIcebergColumnStats(String colName, or try (CloseableIterable fileScanTasks = tableScan.planFiles()) { for (FileScanTask task : fileScanTasks) { int colId = getColId(task.spec(), colName); - totalDataSize += task.file().columnSizes().get(colId); + Map columnSizes = task.file().columnSizes(); + Map nullValueCounts = task.file().nullValueCounts(); + Long columnSize = columnSizes == null ? null : columnSizes.get(colId); + Long nullValueCount = nullValueCounts == null ? null : nullValueCounts.get(colId); + // Iceberg can omit maps or entries for mode=none; partial aggregation would fabricate zero stats. + if (columnSize == null || nullValueCount == null) { + return Optional.empty(); + } + totalDataSize += columnSize; totalDataCount += task.file().recordCount(); - totalNumNull += task.file().nullValueCounts().get(colId); + totalNumNull += nullValueCount; } } catch (IOException e) { LOG.warn("Error to close FileScanTask.", e); + // A failed close can cancel an in-flight empty return, so accumulated stats are not reliable. + return Optional.empty(); } ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(totalDataCount); columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java index 77d318518f326e..39fc15ddbe1a7e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java @@ -18,19 +18,34 @@ package org.apache.doris.datasource.iceberg.helper; import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Test for IcebergWriterHelper DeleteFile conversion @@ -58,6 +73,242 @@ public void setUp() { } + @Test + public void testConvertToWriterResultRespectsNoneMetricsMode() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 128L)); + columnStats.setValueCounts(Map.of(2, 10L)); + columnStats.setNullValueCounts(Map.of(2, 0L)); + columnStats.setLowerBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x01}))); + columnStats.setUpperBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x02}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + WriteResult result = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)); + DataFile dataFile = result.dataFiles()[0]; + + Assertions.assertTrue(dataFile.columnSizes() == null || dataFile.columnSizes().isEmpty()); + Assertions.assertTrue(dataFile.valueCounts() == null || dataFile.valueCounts().isEmpty()); + Assertions.assertTrue(dataFile.nullValueCounts() == null || dataFile.nullValueCounts().isEmpty()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultCountsModeOmitsBounds() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 128L)); + columnStats.setValueCounts(Map.of(2, 10L)); + columnStats.setNullValueCounts(Map.of(2, 0L)); + columnStats.setLowerBounds(Map.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "abcdefgh"))); + columnStats.setUpperBounds(Map.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "ijklmnop"))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(Map.of(2, 128L), dataFile.columnSizes()); + Assertions.assertEquals(Map.of(2, 10L), dataFile.valueCounts()); + Assertions.assertEquals(Map.of(2, 0L), dataFile.nullValueCounts()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultTruncatesStringAndBinaryBounds() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get()), + Types.NestedField.optional(2, "payload", Types.BinaryType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(3)")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "abcdef"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + columnStats.setUpperBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "uvwxyz"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals("abc", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.lowerBounds().get(1)).toString()); + Assertions.assertEquals("uvx", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 3}), dataFile.lowerBounds().get(2)); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 4}), dataFile.upperBounds().get(2)); + } + + @Test + public void testConvertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSuccessor() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(1)")); + + String maxWithoutSuccessor = new String(Character.toChars(Character.MAX_CODE_POINT)) + "tail"; + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setUpperBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), maxWithoutSuccessor))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.orc"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertNotNull(dataFile.upperBounds()); + Assertions.assertEquals(maxWithoutSuccessor, Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + } + + @Test + public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergCommitData firstCommit = new TIcebergCommitData(); + firstCommit.setFilePath("/path/to/first.parquet"); + firstCommit.setRowCount(10); + firstCommit.setFileSize(1024); + + TIcebergCommitData secondCommit = new TIcebergCommitData(); + secondCommit.setFilePath("/path/to/second.parquet"); + secondCommit.setRowCount(20); + secondCommit.setFileSize(2048); + + IcebergWriterHelper.convertToWriterResult(table, List.of(firstCommit, secondCommit)); + + // One schema lookup is made by Iceberg's policy builder and one is captured for all files in the batch. + Mockito.verify(table, Mockito.times(2)).schema(); + } + + @Test + public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@TempDir Path tempDir) { + HadoopTables tables = new HadoopTables(new Configuration()); + Table baseTable = tables.create(schema, unpartitionedSpec, SortOrder.unsorted(), Map.of( + TableProperties.FORMAT_VERSION, "3", + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(16)"), + tempDir.resolve("table").toUri().toString()); + Table transactionTable = baseTable.newTransaction().table(); + + int rowId = MetadataColumns.ROW_ID.fieldId(); + int sequenceNumberId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); + ByteBuffer rowIdBound = Conversions.toByteBuffer(MetadataColumns.ROW_ID.type(), 7L); + ByteBuffer sequenceNumberBound = Conversions.toByteBuffer( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), 3L); + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + columnStats.setUpperBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/v3-data.parquet"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = Assertions.assertDoesNotThrow( + () -> IcebergWriterHelper.convertToWriterResult( + transactionTable, List.of(commitData)).dataFiles()[0]); + Assertions.assertEquals(rowIdBound, dataFile.lowerBounds().get(rowId)); + Assertions.assertEquals(rowIdBound, dataFile.upperBounds().get(rowId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.lowerBounds().get(sequenceNumberId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.upperBounds().get(sequenceNumberId)); + } + + @Test + public void testConvertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields() { + Schema repeatedSchema = new Schema( + Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get())), + Types.NestedField.optional(3, "attributes", + Types.MapType.ofOptional(4, 5, Types.StringType.get(), Types.StringType.get())), + Types.NestedField.optional(6, "top_level", Types.IntegerType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(repeatedSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "full")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L)); + columnStats.setValueCounts(Map.of(2, 2L, 4, 4L, 5, 5L, 6, 6L)); + columnStats.setNullValueCounts(Map.of(2, 0L, 4, 0L, 5, 0L, 6, 0L)); + columnStats.setLowerBounds(Map.of( + 2, Conversions.toByteBuffer(Types.IntegerType.get(), 2), + 4, Conversions.toByteBuffer(Types.StringType.get(), "key"), + 5, Conversions.toByteBuffer(Types.StringType.get(), "value"), + 6, Conversions.toByteBuffer(Types.IntegerType.get(), 6))); + columnStats.setUpperBounds(columnStats.getLowerBounds()); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/repeated.parquet"); + commitData.setRowCount(6); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L), dataFile.columnSizes()); + Assertions.assertEquals(Map.of(6, 6L), dataFile.valueCounts()); + Assertions.assertEquals(Map.of(6, 0L), dataFile.nullValueCounts()); + Assertions.assertEquals(Map.of(6, columnStats.getLowerBounds().get(6)), dataFile.lowerBounds()); + Assertions.assertEquals(Map.of(6, columnStats.getUpperBounds().get(6)), dataFile.upperBounds()); + } + @Test public void testConvertToDeleteFiles_EmptyList() { List commitDataList = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java index 23dcb4403ce731..df9b2ae80c26f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java @@ -74,6 +74,63 @@ public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL)); } + @Test + public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertFalse(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForOrcTopLevelComplexField() throws Exception { + Schema schema = new Schema(Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get()))); + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, schema, Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "items", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { TIcebergDeleteFileDesc deleteFileDesc = new TIcebergDeleteFileDesc(); deleteFileDesc.setPath("file:///tmp/delete.puffin"); @@ -84,13 +141,24 @@ private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { } private static IcebergExternalTable mockIcebergExternalTable(int formatVersion) { - Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + return mockIcebergExternalTable(formatVersion, Collections.emptyMap()); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Map metricsProperties) { + return mockIcebergExternalTable(formatVersion, + new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())), metricsProperties); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Schema schema, Map metricsProperties) { PartitionSpec spec = PartitionSpec.unpartitioned(); Map properties = new HashMap<>(); properties.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); properties.put(TableProperties.DEFAULT_FILE_FORMAT, "parquet"); properties.put(TableProperties.PARQUET_COMPRESSION, "snappy"); properties.put(TableProperties.WRITE_DATA_LOCATION, "file:///tmp/iceberg_tbl/data"); + properties.putAll(metricsProperties); Table icebergTable = Mockito.mock(Table.class); Mockito.when(icebergTable.properties()).thenReturn(properties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 2de5d19e7fb738..16640db7ccddfe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -39,6 +39,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog; +import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; @@ -49,6 +50,8 @@ import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ResultRow; import org.apache.doris.statistics.TableStatsMeta; +import org.apache.doris.thrift.TIcebergColumnStats; +import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Lists; @@ -56,10 +59,20 @@ import mockit.Mock; import mockit.MockUp; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalTime; import java.time.format.DateTimeFormatter; @@ -70,6 +83,63 @@ import java.util.Map; class StatisticsUtilTest { + @Test + void testGetIcebergColumnStatsReturnsEmptyForDisabledMetrics() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(spec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(1, 128L)); + columnStats.setValueCounts(Map.of(1, 10L)); + columnStats.setNullValueCounts(Map.of(1, 0L)); + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()) + .thenReturn(CloseableIterable.withNoopClose(List.of(fileScanTask))); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + + Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + } + + @Test + void testGetIcebergColumnStatsReturnsEmptyWhenCloseFails() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + DataFile dataFile = Mockito.mock(DataFile.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()).thenReturn(CloseableIterable.combine( + List.of(fileScanTask), () -> { + throw new IOException("close failed"); + })); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + Mockito.when(dataFile.columnSizes()).thenReturn(Map.of()); + Mockito.when(dataFile.nullValueCounts()).thenReturn(Map.of()); + + Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + } + @Test void testConvertToDouble() { try { diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 3640117def87ab..a0cb46f11669d9 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -486,6 +486,8 @@ struct TIcebergTableSink { 15: optional map static_partition_values; 16: optional PlanNodes.TSortInfo sort_info; 17: optional TIcebergWriteType write_type = TIcebergWriteType.INSERT; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 18: optional bool collect_column_stats; } struct TIcebergRewritableDeleteFileSet { @@ -532,6 +534,8 @@ struct TIcebergMergeSink { 11: optional map hadoop_config 12: optional Types.TFileType file_type 13: optional list broker_addresses; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 14: optional bool collect_column_stats; // delete side (position delete only) 20: optional TFileContent delete_type From cbd77b921c1e9fcca32807d099c7a1a9fc2d6eb8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 22 Jul 2026 09:21:56 +0800 Subject: [PATCH 14/24] [fix](iceberg) Honor partial name mappings for legacy files (#65784) Issue Number: N/A Related PR: N/A Problem Summary: Legacy Iceberg Parquet and ORC files without field IDs are resolved through name mapping. With a partial mapping, FE omitted the optional mapping metadata for unmapped fields, so both V1 and V2 BE readers fell back to the current physical name and read unrelated data. This change preserves table-level mapping presence by transporting explicit empty per-field lists and makes those lists authoritative in both readers. Unmapped fields now materialize their default or NULL, while mapped aliases and scans without name mapping retain their existing behavior. Fix reads of migrated Iceberg files with partial name mapping so fields omitted from the mapping materialize their default value or NULL instead of matching a physical column by the current name. - Test: Unit Test - `ExternalUtilTest` (6 tests) - Focused V1 and V2 name-mapping BE unit tests under ASAN (10 tests) - Behavior changed: Yes. Partial Iceberg name mappings are now authoritative for legacy files without field IDs. - Does this need documentation: No --- be/src/exec/scan/access_path_parser.cpp | 101 +++- be/src/exec/scan/access_path_parser.h | 6 +- be/src/exec/scan/file_scanner_v2.cpp | 7 +- be/src/format/table/iceberg_scan_semantics.h | 33 ++ be/src/format_v2/column_data.h | 6 + be/src/format_v2/column_mapper.cpp | 133 ++++- be/src/format_v2/column_mapper.h | 4 + ...eberg_position_delete_sys_table_reader.cpp | 31 +- be/src/format_v2/table/iceberg_reader.cpp | 15 +- be/src/format_v2/table/iceberg_reader.h | 36 +- be/src/format_v2/table/iceberg_schema_utils.h | 53 ++ .../format_v2/table/schema_history_util.cpp | 2 + be/src/format_v2/table_reader.cpp | 138 ++++- be/src/format_v2/table_reader.h | 7 +- be/test/exec/scan/access_path_parser_test.cpp | 100 +++- be/test/format_v2/column_mapper_test.cpp | 75 +++ .../format_v2/table/iceberg_reader_test.cpp | 537 +++++++++++++++++- be/test/format_v2/table_reader_test.cpp | 236 +++++++- .../apache/doris/datasource/ExternalUtil.java | 45 +- .../iceberg/IcebergExternalMetaCache.java | 3 +- .../iceberg/IcebergSnapshotCacheValue.java | 20 + .../datasource/iceberg/IcebergUtils.java | 61 +- .../iceberg/source/IcebergScanNode.java | 85 ++- .../doris/datasource/ExternalUtilTest.java | 67 +++ .../iceberg/IcebergDDLAndDMLPlanTest.java | 5 + .../datasource/iceberg/IcebergUtilsTest.java | 14 + .../iceberg/source/IcebergScanNodeTest.java | 325 ++++++++++- gensrc/thrift/ExternalTableSchema.thrift | 5 +- gensrc/thrift/PlanNodes.thrift | 4 + 29 files changed, 1973 insertions(+), 181 deletions(-) create mode 100644 be/src/format/table/iceberg_scan_semantics.h create mode 100644 be/src/format_v2/table/iceberg_schema_utils.h diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..1c48b9285d9eda 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,10 +87,18 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + // The presence bit is part of the mapping contract: an explicit empty mapping must remain + // authoritative after access-path pruning instead of enabling current-name fallback. + column->has_name_mapping = schema_column->has_name_mapping; + // Initial defaults describe the logical value of fields absent from older files. Nested + // access-path pruning must retain them just like it retains rename metadata. + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( - const format::ColumnDefinition* schema_column, const std::string& child_path) { + const format::ColumnDefinition* schema_column, const std::string& child_path, + bool prefer_exact_name_match) { if (schema_column == nullptr) { return nullptr; } @@ -103,15 +111,31 @@ const format::ColumnDefinition* find_schema_child_by_path( }); return child_it == schema_column->children.end() ? nullptr : &*child_it; } - const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { - if (to_lower(child.name) == to_lower(child_path)) { - return true; - } + if (!prefer_exact_name_match) { + const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + if (to_lower(child.name) == to_lower(child_path)) { + return true; + } + return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(child_path); + }); + }); + return child_it == schema_column->children.end() ? nullptr : &*child_it; + } + // Iceberg can reuse a historical name for a newly added sibling. Current names therefore + // have precedence across the entire struct; an earlier alias must not steal that access path. + const auto exact_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + return to_lower(child.name) == to_lower(child_path); + }); + if (exact_it != schema_column->children.end()) { + return &*exact_it; + } + const auto alias_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { return to_lower(alias) == to_lower(child_path); }); }); - return child_it == schema_column->children.end() ? nullptr : &*child_it; + return alias_it == schema_column->children.end() ? nullptr : &*alias_it; } int32_t schema_field_id(const format::ColumnDefinition* schema_column) { @@ -169,7 +193,8 @@ void insert_access_path(AccessPathNode* root, const std::vector& pa Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match); // Expand a full complex-column projection into table-schema children when the table format provides // an external/current schema. Without this, `SELECT complex_col` or `SELECT *` leaves @@ -186,7 +211,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, // wrapper here: ColumnMapper and TableReader treat MAP children as [key, value]. Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const DataTypePtr& type, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); const auto nested_type = remove_nullable(type); @@ -197,14 +223,16 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const auto& struct_type = assert_cast(*nested_type); for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { const auto field_name = struct_type.get_element_name(field_idx); - const auto* schema_child = find_schema_child_by_path(schema_column, field_name); + const auto* schema_child = + find_schema_child_by_path(schema_column, field_name, prefer_exact_name_match); auto* child = find_or_add_child( column, schema_field_id_or(schema_child, cast_set(field_idx)), schema_field_name_or(schema_child, field_name), struct_type.get_element(field_idx)); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, project_all, path + "." + child->name, schema_child)); + child, child->type, project_all, path + "." + child->name, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -217,7 +245,7 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, project_all, path + ".*", - element_schema); + element_schema, prefer_exact_name_match); } case TYPE_MAP: { const auto& map_type = assert_cast(*nested_type); @@ -231,12 +259,14 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - key_child, key_child->type, project_all, path + ".KEYS", key_schema)); + key_child, key_child->type, project_all, path + ".KEYS", key_schema, + prefer_exact_name_match)); auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, project_all, path + ".VALUES", value_schema)); + value_child, value_child->type, project_all, path + ".VALUES", value_schema, + prefer_exact_name_match)); return Status::OK(); } default: @@ -247,7 +277,8 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, Status build_struct_children_from_access_node(format::ColumnDefinition* column, const DataTypeStruct& struct_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); for (const auto& [child_path, child_node] : node.children) { // Struct children are resolved by name or schema field id. We do not treat a numeric @@ -262,7 +293,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, // Prefer the table/schema ColumnDefinition because it carries field ids and aliases. // Fallback to the struct type name only for formats without external schema metadata. - const auto* schema_child = find_schema_child_by_path(schema_column, child_path); + const auto* schema_child = + find_schema_child_by_path(schema_column, child_path, prefer_exact_name_match); int32_t field_id = schema_field_id(schema_child); std::string field_name = schema_child == nullptr ? child_path : schema_child->name; DataTypePtr field_type = schema_child == nullptr ? nullptr : schema_child->type; @@ -288,7 +320,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, auto* child = find_or_add_child(column, field_id, field_name, field_type); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, child_node, path + "." + child_path, schema_child)); + child, child->type, child_node, path + "." + child_path, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -296,7 +329,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, Status build_map_children_from_access_node(format::ColumnDefinition* column, const DataTypeMap& map_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); AccessPathNode key_node; AccessPathNode value_node; @@ -368,14 +402,16 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node(key_child, key_child->type, key_node, - path + ".KEYS", key_schema)); + path + ".KEYS", key_schema, + prefer_exact_name_match)); } if (need_value) { auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, value_node, path + ".VALUES", value_schema)); + value_child, value_child->type, value_node, path + ".VALUES", value_schema, + prefer_exact_name_match)); } return Status::OK(); } @@ -383,18 +419,20 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (node.project_all || node.children.empty()) { - return build_all_nested_children_from_schema(column, type, path, schema_column); + return build_all_nested_children_from_schema(column, type, path, schema_column, + prefer_exact_name_match); } const auto nested_type = remove_nullable(type); switch (nested_type->get_primitive_type()) { case TYPE_STRUCT: return build_struct_children_from_access_node( - column, assert_cast(*nested_type), node, path, - schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); case TYPE_ARRAY: { if (node.children.size() != 1 || !node.children.contains("*")) { return Status::NotSupported( @@ -409,11 +447,13 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, node.children.at("*"), - path + ".*", element_schema); + path + ".*", element_schema, + prefer_exact_name_match); } case TYPE_MAP: return build_map_children_from_access_node( - column, assert_cast(*nested_type), node, path, schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); default: return Status::NotSupported("AccessPathParser does not support access path {} for slot {}", path, column->name); @@ -424,7 +464,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (is_scanner_materialized_virtual_column(column->name)) { return Status::OK(); @@ -465,15 +506,17 @@ Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, } // Recursively build nested children for the column based on the AccessPathNode tree. return build_nested_children_from_access_node(column, column->type, root, column->name, - schema_column); + schema_column, prefer_exact_name_match); } Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); DORIS_CHECK(slot_desc != nullptr); - return build_nested_children(column, slot_desc->all_access_paths(), schema_column); + return build_nested_children(column, slot_desc->all_access_paths(), schema_column, + prefer_exact_name_match); } } // namespace doris diff --git a/be/src/exec/scan/access_path_parser.h b/be/src/exec/scan/access_path_parser.h index 1aa4c5b89d492a..0be785a33906e8 100644 --- a/be/src/exec/scan/access_path_parser.h +++ b/be/src/exec/scan/access_path_parser.h @@ -31,11 +31,13 @@ class AccessPathParser { public: static Status build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); static Status build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); }; } // namespace doris diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index daf179e36f1fe7..b0c08bcce27552 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -47,6 +47,7 @@ #include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" +#include "format/table/iceberg_scan_semantics.h" #include "format_v2/column_mapper.h" #include "format_v2/jni/iceberg_sys_table_reader.h" #include "format_v2/jni/jdbc_reader.h" @@ -682,6 +683,9 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ .range = &_current_range, .runtime_state = _state, }; + // Field 34 is the rollout boundary for root and nested exact-name precedence. + const bool prefer_exact_name_match = + !_params->__isset.history_schema_info || supports_iceberg_scan_semantics_v1(_params); for (size_t slot_idx = 0; slot_idx < _params->required_slots.size(); ++slot_idx) { const auto& slot_info = _params->required_slots[slot_idx]; @@ -702,7 +706,8 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ // column's nested children. RETURN_IF_ERROR(AccessPathParser::build_nested_children( &column, it->second, - build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr)); + build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr, + prefer_exact_name_match)); if (is_partition_slot(slot_info, column.name)) { column.is_partition_key = true; _partition_slot_descs.emplace( diff --git a/be/src/format/table/iceberg_scan_semantics.h b/be/src/format/table/iceberg_scan_semantics.h new file mode 100644 index 00000000000000..f579f063b76327 --- /dev/null +++ b/be/src/format/table/iceberg_scan_semantics.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +inline constexpr int32_t ICEBERG_SCAN_SEMANTICS_VERSION_1 = 1; + +inline bool supports_iceberg_scan_semantics_v1(const TFileScanRangeParams* params) { + // Old FE plans can carry IDs and encoded defaults too, so only this explicit version marker + // may opt a new BE into result-changing semantics during a rolling upgrade. + return params != nullptr && params->__isset.iceberg_scan_semantics_version && + params->iceberg_scan_semantics_version >= ICEBERG_SCAN_SEMANTICS_VERSION_1; +} + +} // namespace doris diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index ac510caaf07ac4..d504305fad4014 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -248,6 +248,9 @@ struct ColumnDefinition { // Historical or external names for the same logical field. Table formats such as Iceberg can // use this to resolve partition path keys after column rename. std::vector name_mapping {}; + // Distinguishes no table-level mapping from an explicit empty field mapping. The latter must + // not fall back to the current name when matching fields in legacy files. + bool has_name_mapping = false; DataTypePtr type; // Semantic nested children for this schema node. // @@ -255,6 +258,9 @@ struct ColumnDefinition { // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For // example, MAP children are key/value and ARRAY children contain only the element field. std::vector children {}; + // Full table-schema identity subtree before access-path pruning. ID-less physical complex + // wrappers must be discovered from this view without adding unrequested children to output. + std::vector identity_children {}; // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 4c23bc9f83380b..e2d1fba8b3a8b4 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -52,11 +52,77 @@ #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "util/url_coding.h" namespace doris::format { namespace { +Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { + DORIS_CHECK(value != nullptr); + *value = nullptr; + if (!column.initial_default_value.has_value()) { + return Status::OK(); + } + const auto nested_type = remove_nullable(column.type); + Field parsed; + if (column.initial_default_value_is_base64 || + nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(*column.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + column.name); + } + parsed = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // Variable-width Fields borrow their input. Materialize while decoded is alive so the + // resulting column owns the payload before it crosses a mapping/literal boundary. + *value = column.type->create_column_const(1, parsed); + return Status::OK(); + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); + } + *value = column.type->create_column_const(1, parsed); + return Status::OK(); +} + +Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { + DORIS_CHECK(literal != nullptr); + ColumnPtr owned_value; + RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); + DORIS_CHECK(static_cast(owned_value)); + Field value; + owned_value->get(0, value); + // VLiteral copies the borrowed Field into its own column while owned_value is still alive. + *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); + return Status::OK(); +} + +bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { + const auto& table_children = + table.identity_children.empty() ? table.children : table.identity_children; + for (const auto& table_child : table_children) { + if (!table_child.has_identifier_field_id()) { + continue; + } + const auto file_child = + std::ranges::find_if(file.children, [&](const ColumnDefinition& candidate) { + return candidate.has_identifier_field_id() && + candidate.get_identifier_field_id() == + table_child.get_identifier_field_id(); + }); + if (file_child != file.children.end() || + std::ranges::any_of(file.children, [&](const ColumnDefinition& candidate) { + return has_shared_descendant_field_id(table_child, candidate); + })) { + return true; + } + } + return false; +} + std::string mapping_mode_to_string(TableColumnMappingMode mode) { switch (mode) { case TableColumnMappingMode::BY_FIELD_ID: @@ -82,12 +148,16 @@ bool column_has_name(const ColumnDefinition& column, const std::string& name) { } bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { - if (column_has_name(rhs, lhs.name)) { - return true; - } - if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { - return true; + if (!lhs.has_name_mapping) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } } + // Explicit Iceberg name mapping is authoritative: an empty alias list represents a field that + // did not exist in the imported file, so only transported aliases may match. return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { return column_has_name(rhs, alias); }); @@ -314,7 +384,9 @@ static bool is_binary_comparison_predicate(const VExprSPtr& expr) { std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; - out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection + << "}"; return out.str(); } @@ -323,9 +395,13 @@ std::string ColumnDefinition::debug_string() const { out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) << ", name_mapping=" << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) - << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << ", has_name_mapping=" << has_name_mapping << ", local_id=" << local_id + << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", identity_children=" + << join_debug_strings(identity_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) << ", is_partition_key=" << is_partition_key << "}"; return out.str(); @@ -938,7 +1014,8 @@ static bool rewrite_binary_struct_literal_predicate( const auto table_leaf_type = children[struct_child_idx]->data_type(); DORIS_CHECK(table_leaf_type != nullptr); - auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type); + auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type, + rewrite_context); if (table_literal == nullptr || !rewrite_struct_element_path_to_file_expr(children[struct_child_idx], filter_mappings, global_to_file_slot, rewrite_context)) { @@ -992,7 +1069,8 @@ static bool rewrite_in_struct_literal_predicate( VExprSPtrs table_literals; table_literals.reserve(children.size() - 1); for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { - auto table_literal = unwrap_literal_for_file_cast(children[child_idx], table_leaf_type); + auto table_literal = + unwrap_literal_for_file_cast(children[child_idx], table_leaf_type, rewrite_context); if (table_literal == nullptr) { return false; } @@ -1981,6 +2059,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.initial_default_value.has_value()) { + VExprContextSPtr initial_default; + RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); + // Iceberg metadata is the authoritative logical value for files written before the field + // existed; the generic FE expression may still contain its Base64 transport text. + _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); @@ -2303,7 +2387,25 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( }); return field_it == file_schema.end() ? nullptr : &*field_it; } - return matcher_for_mode(_options.mode).find(table_column, file_schema); + const auto* matched = matcher_for_mode(_options.mode).find(table_column, file_schema); + if (matched != nullptr || _options.mode != TableColumnMappingMode::BY_FIELD_ID || + !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { + return matched; + } + const ColumnDefinition* wrapper = nullptr; + for (const auto& candidate : file_schema) { + if (candidate.has_identifier_field_id() || candidate.children.empty() || + !has_shared_descendant_field_id(table_column, candidate)) { + continue; + } + if (wrapper != nullptr) { + return nullptr; + } + wrapper = &candidate; + } + // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is + // selected. Descendant IDs, not aliases, identify that wrapper; ambiguity remains unmapped. + return wrapper; } Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, @@ -2341,6 +2443,13 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c const auto* file_child = find_file_child_for_mapping(table_child, file_field, _options.mode, table_child_idx, synthesized_table_children); + if (file_child == nullptr && !synthesized_table_children && + _options.mode == TableColumnMappingMode::BY_FIELD_ID && + _options.allow_idless_complex_wrapper_projection) { + // Parquet can retain an ID-less wrapper at any depth when a selected descendant + // has an ID; apply the same opt-in fallback used for root lookup recursively. + file_child = _find_file_field(table_child, file_field.children); + } if (synthesized_table_children && file_child != nullptr) { const auto file_child_id = file_child->file_local_id(); if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != @@ -2366,6 +2475,10 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + // A missing nested field still has its Iceberg initial-default value in every row + // written before the field was added; carry it into recursive materialization. + RETURN_IF_ERROR(build_initial_default_column( + table_child, &child_mapping.initial_default_column)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 4f417a680aad60..0b42ac39a5cbea 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -150,12 +150,16 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; + // One-row constant owns variable-width payloads; Field is only a borrowed + // StringView and cannot safely outlive the Base64 decode buffer used to construct it. + ColumnPtr initial_default_column; std::string debug_string() const; }; struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + bool allow_idless_complex_wrapper_projection = false; std::string debug_string() const; }; diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..2f808188f0d7c2 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -35,6 +35,7 @@ #include "core/types.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -132,24 +133,24 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) { } } -bool has_field_id(const std::vector& schema) { - for (const auto& field : schema) { - if (!field.has_identifier_field_id()) { - return false; - } - if (!has_field_id(field.children)) { - return false; - } - } - return true; -} - class PositionDeleteFileTableReader final : public format::TableReader { protected: format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; + } + + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + // Parquet may preserve a selected complex wrapper without its own ID; position-delete row + // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && + _format == format::FileFormat::PARQUET; } }; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index fcbefcba1f0d81..d83a8c63dc8547 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -91,7 +91,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; + VExprSPtr literal; if (table_field.initial_default_value_is_base64 || table_field.type->get_primitive_type() == TYPE_VARBINARY) { // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its @@ -104,19 +104,26 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit table_field.name); } if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); + const auto initial_default = + Field::create_field(StringView(decoded_default)); + // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and + // long FIXED defaults otherwise retain a pointer into freed decode storage. + literal = VLiteral::create_shared(table_field.type, initial_default); } else { DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); + literal = VLiteral::create_shared(table_field.type, + Field::create_field(decoded_default)); } } else { // An added field's initial default is its logical value in every older data file that lacks // the physical column. FE normalizes the string for the current Doris table type. + Field initial_default; RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( *table_field.initial_default_value, initial_default)); + literal = VLiteral::create_shared(table_field.type, initial_default); } - auto literal = VLiteral::create_shared(table_field.type, initial_default); + DORIS_CHECK(literal != nullptr); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 79e0174a3d7eaa..dde136e0de5763 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -27,6 +27,7 @@ #include "core/block/block.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format_v2/file_reader.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" @@ -59,12 +60,21 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } protected: + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET; + } + Status materialize_virtual_columns(Block* table_block) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; @@ -78,26 +88,6 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - - bool _has_field_id(const std::vector& schema) const { - for (const auto& field : schema) { - // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the - // first-phase scan. That virtual column is not an Iceberg data field and therefore has - // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, - // otherwise old data files whose physical names predate a rename (for example, - // table column `new_new_id` stored as file column `id`) are materialized as defaults. - if (field.column_type != format::ColumnType::DATA_COLUMN) { - continue; - } - if (!field.has_identifier_field_id()) { - return false; - } - if (!_has_field_id(field.children)) { - return false; - } - } - return true; - } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; static constexpr int EQUALITY_DELETE = 2; diff --git a/be/src/format_v2/table/iceberg_schema_utils.h b/be/src/format_v2/table/iceberg_schema_utils.h new file mode 100644 index 00000000000000..516c982194bbe4 --- /dev/null +++ b/be/src/format_v2/table/iceberg_schema_utils.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format/table/iceberg_scan_semantics.h" +#include "format_v2/column_data.h" + +namespace doris::format::iceberg { + +inline bool schema_has_any_field_id(const std::vector& schema) { + for (const auto& field : schema) { + // Iceberg's hasIds contract is existential for the whole file. Ignore synthesized columns + // and retain ID projection when any real field, including a nested field, carries an ID. + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (field.has_identifier_field_id() || schema_has_any_field_id(field.children)) { + return true; + } + } + return false; +} + +inline bool schema_has_all_field_ids(const std::vector& schema) { + for (const auto& field : schema) { + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (!field.has_identifier_field_id() || !schema_has_all_field_ids(field.children)) { + return false; + } + } + return true; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 10109839e6987d..440211607d633d 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,6 +77,8 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; + column->has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 40654de69dddeb..8ae0a70242963c 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -40,6 +40,7 @@ #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" #include "format_v2/deletion_vector_reader.h" @@ -143,6 +144,93 @@ const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& return field_ptr.field_ptr.get(); } +ColumnDefinition build_schema_identity_from_external_field(const schema::external::TField& field) { + ColumnDefinition identity; + if (field.__isset.id) { + identity.identifier = Field::create_field(field.id); + } + identity.name = field.__isset.name ? field.name : ""; + identity.name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + identity.has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; + if (!field.__isset.nestedField) { + return identity; + } + if (field.nestedField.__isset.struct_field && field.nestedField.struct_field.__isset.fields) { + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + if (const auto* child = get_field_ptr(child_ptr); child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + } + } + } else if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field) { + if (const auto* child = get_field_ptr(field.nestedField.array_field.item_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "element"; + } + } else if (field.nestedField.__isset.map_field) { + if (field.nestedField.map_field.__isset.key_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.key_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "key"; + } + } + if (field.nestedField.map_field.__isset.value_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.value_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "value"; + } + } + } + return identity; +} + +const ColumnDefinition* find_identity_child(const ColumnDefinition& projected_child, + const ColumnDefinition& identity_parent) { + const auto child_it = std::ranges::find_if( + identity_parent.children, [&](const ColumnDefinition& identity_child) { + if (projected_child.has_identifier_field_id() && + identity_child.has_identifier_field_id()) { + return projected_child.get_identifier_field_id() == + identity_child.get_identifier_field_id(); + } + if (to_lower(projected_child.name) == to_lower(identity_child.name)) { + return true; + } + return std::ranges::any_of( + identity_child.name_mapping, [&](const std::string& alias) { + return to_lower(projected_child.name) == to_lower(alias); + }); + }); + return child_it == identity_parent.children.end() ? nullptr : &*child_it; +} + +void attach_full_schema_identity(ColumnDefinition* projected, const ColumnDefinition& identity) { + DORIS_CHECK(projected != nullptr); + // Access-path children control materialization, but wrapper discovery needs sibling IDs that + // were pruned from that projection. Keep the complete identity tree on a separate channel. + projected->identity_children = identity.children; + for (auto& projected_child : projected->children) { + if (const auto* identity_child = find_identity_child(projected_child, identity); + identity_child != nullptr) { + attach_full_schema_identity(&projected_child, *identity_child); + } + } +} + +void clear_initial_default_metadata(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + column->initial_default_value.reset(); + column->initial_default_value_is_base64 = false; + for (auto& child : column->children) { + clear_initial_default_metadata(&child); + } +} + bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { if (field.__isset.name && to_lower(field.name) == to_lower(name)) { return true; @@ -189,6 +277,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .has_name_mapping = field.__isset.name_mapping_is_authoritative && + field.name_mapping_is_authoritative, .type = std::move(type), .children = {}, .default_expr = nullptr, @@ -298,12 +388,32 @@ const schema::external::TField* find_external_root_field(const TFileScanRangePar if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { return nullptr; } + if (!supports_iceberg_scan_semantics_v1(params)) { + // Old BEs used one ordered current-name/alias pass. Preserve that result for old-FE plans + // until the explicit scan-semantics marker makes exact-name precedence cluster-wide. + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && external_field_matches_name(*field, column.name)) { + return field; + } + } + return nullptr; + } + // A reused name identifies the newly added field, not an older sibling that retained that + // spelling as an alias. Exhaust exact current names before consulting historical aliases. for (const auto& field_ptr : schema->root_field.fields) { const auto* field = get_field_ptr(field_ptr); - if (field == nullptr) { - continue; + if (field != nullptr && field->__isset.name && + to_lower(field->name) == to_lower(column.name)) { + return field; } - if (external_field_matches_name(*field, column.name)) { + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.name_mapping && + std::ranges::any_of(field->name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(column.name); + })) { return field; } } @@ -486,8 +596,20 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); + const bool use_current_semantics = supports_iceberg_scan_semantics_v1(context->scan_params); + if (!use_current_semantics) { + // IDs and encoded defaults predate the result-changing semantics. Strip only the new + // default channel so an old-FE plan keeps the same generic root/nested values on every BE. + clear_initial_default_metadata(&*context->schema_column); + } column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; + column->has_name_mapping = context->schema_column->has_name_mapping; + // Projected roots already carry a generic FE default expression, but Iceberg binary defaults + // need the raw Base64 marker so missing-file materialization can decode rather than copy text. + column->initial_default_value = context->schema_column->initial_default_value; + column->initial_default_value_is_base64 = + context->schema_column->initial_default_value_is_base64; return Status::OK(); } @@ -531,6 +653,16 @@ Status TableReader::init(TableReadOptions&& options) { _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; _projected_columns = std::move(options.projected_columns); + if (supports_iceberg_scan_semantics_v1(_scan_params)) { + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity( + &projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } + } _system_properties = create_system_properties(_scan_params); _mapper_options.mode = TableColumnMappingMode::BY_NAME; _conjuncts = std::move(options.conjuncts); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index b9683de564ec5b..c26ba086788020 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -369,6 +369,7 @@ class TableReader { Status create_next_reader(bool* eos); virtual Status create_file_reader(std::unique_ptr* reader); virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { DORIS_CHECK(file_schema != nullptr); return Status::OK(); @@ -385,6 +386,7 @@ class TableReader { RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); + configure_mapper_options(&_mapper_options); _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); DORIS_CHECK(_data_reader.column_mapper != nullptr); @@ -1368,7 +1370,10 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) + (child_mapping->initial_default_column + ? child_mapping->initial_default_column->clone_resized(rows) + : child_mapping->table_type + ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); continue; } diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..02cf3013139cd3 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -59,11 +59,13 @@ TColumnAccessPath meta_access_path() { format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, std::vector children = {}, - std::vector aliases = {}) { + std::vector aliases = {}, + bool has_name_mapping = false) { return { .identifier = Field::create_field(id), .name = std::move(name), .name_mapping = std::move(aliases), + .has_name_mapping = has_name_mapping, .type = std::move(type), .children = std::move(children), }; @@ -167,7 +169,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { .type = struct_type, .children = { - field(101, "a", int_type), + field(101, "a", int_type, {}, {}, true), field(205, "b", int_type, {}, {"old_b"}), }, }; @@ -199,6 +201,15 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { expect_child(column.children[0], 205, "b"); EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "a"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); + } { auto column = root_column(100, "s", struct_type); auto status = AccessPathParser::build_nested_children( @@ -206,6 +217,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); expect_child(column.children[1], 205, "b"); } @@ -218,6 +230,52 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { } } +TEST(AccessPathParserTest, CurrentStructNameTakesPriorityOverHistoricalAlias) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 2, "b"); +} + +TEST(AccessPathParserTest, LegacyPlanRetainsOrderedStructNameAndAliasLookup) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema, false); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 1, "renamed_b"); +} + // Scenario: array access paths must pass through the "*" element token, then reuse struct child // parsing under the element wrapper; invalid array tokens are rejected. TEST(AccessPathParserTest, ArrayAccessPathMatrix) { @@ -235,7 +293,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { field(201, "element", element_type, { field(202, "item", string_type, {}, {"old_item"}), - field(203, "quantity", int_type), + field(203, "quantity", int_type, {}, {}, true), }), }, }; @@ -254,6 +312,18 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { EXPECT_EQ(column.children[0].children[0].name_mapping, std::vector({"old_item"})); } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "quantity"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[0].has_name_mapping); + } { auto column = root_column(200, "items", array_type); auto status = AccessPathParser::build_nested_children( @@ -264,6 +334,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { ASSERT_EQ(column.children[0].children.size(), 2); expect_child(column.children[0].children[0], 202, "item"); expect_child(column.children[0].children[1], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[1].has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -293,7 +364,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { field(302, "value", value_type, { field(303, "full_name", string_type, {}, {"name"}), - field(304, "age", int_type), + field(304, "age", int_type, {}, {}, true), field(305, "gender", string_type), }), }, @@ -329,6 +400,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { expect_child(column.children[1], 302, "value"); ASSERT_EQ(column.children[1].children.size(), 1); expect_child(column.children[1].children[0], 304, "age"); + EXPECT_TRUE(column.children[1].children[0].has_name_mapping); } { auto column = root_column(300, "m", map_type); @@ -357,6 +429,9 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); ASSERT_EQ(column.children[1].children.size(), 3); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + EXPECT_TRUE(age->has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -368,4 +443,21 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { } } +TEST(AccessPathParserTest, PreservesNestedInitialDefaultMetadata) { + auto binary_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {binary_type}, Strings {"data"}); + auto defaulted_child = field(101, "data", binary_type); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto schema = field(100, "s", struct_type, {defaulted_child}); + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "data"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].initial_default_value, std::optional("AAEC/w==")); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); +} + } // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 1a21538544a8b2..e134248f814081 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -255,6 +255,81 @@ TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { } } +TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { + auto table_child = field_id_col("a", 1, i32()); + auto table_struct = struct_col("s", 10, {table_child}); + auto file_child = field_id_col("legacy_a", 1, i32(), 0); + auto file_struct = struct_name_col("s", {file_child}, 0); + + TableColumnMapper parquet_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(parquet_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(parquet_mapper.mappings().size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(parquet_mapper.mappings()[0].child_mappings.size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].child_mappings[0].file_local_id, 0); + + TableColumnMapper orc_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(orc_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + auto table_struct = struct_col("s", 10, {field_id_col("a", 2, i32())}); + table_struct.has_name_mapping = true; + auto file_struct = struct_name_col("s", {field_id_col("a", 2, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + EXPECT_TRUE(mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { + auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); + auto table_outer = struct_col("outer", 10, {table_inner}); + auto file_inner = struct_name_col("inner", {field_id_col("leaf", 30, i32(), 0)}, 0); + auto file_outer = struct_col("outer", 10, {file_inner}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_outer}, {}, {file_outer}).ok()); + + const auto& outer_mapping = mapper.mappings()[0]; + ASSERT_TRUE(outer_mapping.file_local_id.has_value()); + ASSERT_EQ(outer_mapping.child_mappings.size(), 1); + const auto& inner_mapping = outer_mapping.child_mappings[0]; + ASSERT_TRUE(inner_mapping.file_local_id.has_value()); + ASSERT_EQ(inner_mapping.child_mappings.size(), 1); + EXPECT_TRUE(inner_mapping.child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { + auto defaulted_child = field_id_col("data", 2, varbinary()); + defaulted_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + defaulted_child.initial_default_value_is_base64 = true; + auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); + auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + const auto& missing = mapper.mappings()[0].child_mappings[1]; + ASSERT_TRUE(missing.initial_default_column); + Field value; + missing.initial_default_column->get(0, value); + EXPECT_EQ(value.get_type(), TYPE_VARBINARY); + EXPECT_EQ(std::string(value.get()), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + void expect_mapping(const ColumnMapping& mapping, size_t global_index, const std::string& table_name, int32_t file_local_id, const std::string& file_name, const DataTypePtr& file_type, diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 6253ab853a8c3b..b7af62c400251d 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -56,6 +56,7 @@ #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" +#include "exec/scan/access_path_parser.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" @@ -231,7 +232,9 @@ class IcebergTableReaderScanRequestTestHelper final class IcebergTableReaderMappingModeTestHelper final : public doris::format::iceberg::IcebergTableReader { public: - TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema) { + TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema, + TFileScanRangeParams* scan_params = nullptr) { + _scan_params = scan_params; _data_reader.file_schema = std::move(file_schema); return mapping_mode(); } @@ -517,6 +520,77 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value, + bool outer_has_field_id = true) { + const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); + auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); + auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); + ASSERT_TRUE(inner_result.ok()) << inner_result.status(); + auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); + auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false); + if (outer_has_field_id) { + outer_field = + outer_field->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + } + auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_nullable_idless_struct_with_sibling_id_parquet_file(const std::string& file_path) { + const auto sibling_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto sibling_field = arrow::field("b", arrow::int32(), false)->WithMetadata(sibling_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 42})}, {sibling_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = arrow::field("s", arrow::struct_({sibling_field}), true); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + +void write_nullable_renamed_struct_child_parquet_file(const std::string& file_path) { + const auto child_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + auto child_field = arrow::field("b", arrow::int32(), false)->WithMetadata(child_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 10})}, {child_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), true) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -883,9 +957,19 @@ TFileScanRangeParams make_local_parquet_scan_params() { TFileScanRangeParams scan_params; scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); return scan_params; } +TColumnAccessPath nested_data_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_path; + data_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_path)); + return access_path; +} + std::shared_ptr make_io_context(io::FileReaderStats* file_reader_stats, io::FileCacheStatistics* file_cache_stats) { auto io_ctx = std::make_shared(); @@ -1858,10 +1942,12 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { TableColumnMappingMode::BY_FIELD_ID); } -// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data -// column without an Iceberg field id still disables field-id mapping. -TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { +// Iceberg treats a schema as ID-bearing when any physical data field has an ID. Keeping ID mode +// preserves authoritative matches even when a sibling was written without an ID. +TEST(IcebergV2ReaderTest, IcebergMappingModeUsesAnyDataColumnFieldId) { IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); std::vector file_schema { make_file_column(1, "id", std::make_shared()), make_file_column(2, "name", std::make_shared()), @@ -1869,7 +1955,35 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { }; file_schema[1].identifier = Field {}; - EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), + EXPECT_EQ(reader.mapping_mode_for_schema(file_schema, &scan_params), + TableColumnMappingMode::BY_FIELD_ID); + + file_schema[0].identifier = Field {}; + file_schema[0].children.emplace_back( + make_file_column(3, "nested", std::make_shared())); + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &scan_params), + TableColumnMappingMode::BY_FIELD_ID); +} + +TEST(IcebergV2ReaderTest, IcebergLegacyPlanKeepsAllFieldIdsMappingRule) { + IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams old_fe_scan_params; + std::vector file_schema { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + file_schema[1].identifier = Field {}; + + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &old_fe_scan_params), + TableColumnMappingMode::BY_NAME); + + auto nested = make_file_column(10, "s", std::make_shared()); + nested.children = { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + nested.children[1].identifier = Field {}; + EXPECT_EQ(reader.mapping_mode_for_schema({std::move(nested)}, &old_fe_scan_params), TableColumnMappingMode::BY_NAME); } @@ -2297,7 +2411,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + const std::string binary_default( + "\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2306,10 +2421,10 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "Ej5FZ+ibEtOkVkJmFBdAAA==", + external_primitive_type(TPrimitiveType::VARBINARY, 16), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -2500,7 +2615,401 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { +TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_recursive_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42, false); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + inner.has_name_mapping = true; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + outer.has_name_mapping = true; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_unprojected_sibling_id_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.initial_default_value = "7"; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + auto schema_a = + external_schema_field("a", 1, {}, "7", external_primitive_type(TPrimitiveType::INT)); + auto schema_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + schema::external::TStructField struct_children; + struct_children.__set_fields({schema_a, schema_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedRootNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_single_int_parquet_file(file_path, "b", {10}, 1); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({7})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanRetainsOrderedRootNameAndAliasLookupWithAllFieldIds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_legacy_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_two_int_parquet_file(file_path, "renamed_b", {10}, 1, "b", {20}, 2); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + ASSERT_EQ(projected_b.get_identifier_field_id(), 1); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedNestedNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_nested_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_renamed_struct_child_parquet_file(file_path); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + schema::external::TStructField struct_children; + struct_children.__set_fields({renamed_b, current_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + + const auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"b"}); + auto projected_s = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_s).ok()); + ASSERT_TRUE(context.schema_column.has_value()); + ASSERT_TRUE(AccessPathParser::build_nested_children( + &projected_s, + std::vector {nested_data_access_path({"s", "b"})}, + &*context.schema_column) + .ok()); + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_b = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_b.is_null_at(1)); + const auto& values = assert_cast(nullable_b.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanUsesNestedNameMappingForMixedFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_legacy_nested_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.name_mapping = {"b"}; + projected_a.has_name_mapping = true; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; std::filesystem::remove_all(test_dir); @@ -2508,8 +3017,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); - // The real key has no id and is mapped by its historical name. A different physical column - // carries the stale id 0, forcing the entire split into BY_NAME mode. + // Iceberg's existential hasIds contract makes the physical field carrying ID 0 authoritative; + // an ID-less alias cannot switch a mixed schema back to name projection. write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", {100, 200, 300}, 0); write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); @@ -2534,7 +3043,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({100, 200, 300})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 1c7bb903ac7b20..ddc52f41cdc5e0 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -41,6 +41,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -48,11 +49,13 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" #include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" +#include "format/table/iceberg_scan_semantics.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -2181,6 +2184,224 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, AnnotateProjectedColumnPrefersCurrentNameOverHistoricalAlias) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 2); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_TRUE(projected.name_mapping.empty()); +} + +TEST(TableReaderTest, LegacyPlanRetainsOrderedCurrentNameAndAliasLookup) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info( + {external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 1); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_EQ(projected.name_mapping, std::vector({"b"})); +} + +TEST(TableReaderTest, IcebergInitialDefaultMetadataOverridesGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_top_level_binary_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto int_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); + auto id_column = make_table_column(-1, "id", int_type); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + ASSERT_TRUE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(binary_column.initial_default_value_is_base64); + + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, IcebergLegacyPlanKeepsGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_legacy_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(1); + old_fe_scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto varbinary_type = std::make_shared(16); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + + EXPECT_FALSE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(context.schema_column.has_value()); + EXPECT_FALSE(context.schema_column->initial_default_value.has_value()); + + auto nested_child = external_schema_field("added_binary", 2); + nested_child.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + nested_child.field_ptr->__set_initial_default_value_is_base64(true); + auto nested_field = external_struct_field("s", 10, {nested_child}); + TFileScanRangeParams old_fe_nested_params; + old_fe_nested_params.__set_current_schema_id(1); + old_fe_nested_params.__set_history_schema_info({external_schema(1, {nested_field})}); + auto struct_type = + std::make_shared(DataTypes {varbinary_type}, Strings {"added_binary"}); + auto struct_column = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext nested_context {.scan_params = &old_fe_nested_params}; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &nested_context, &struct_column).ok()); + ASSERT_TRUE(nested_context.schema_column.has_value()); + ASSERT_EQ(nested_context.schema_column->children.size(), 1); + EXPECT_FALSE(nested_context.schema_column->children[0].initial_default_value.has_value()); + + auto id_column = make_table_column(-1, "id", std::make_shared()); + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader data_reader; + ASSERT_TRUE(data_reader + .init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &old_fe_scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(data_reader.prepare_split(build_split_options(file_path)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(data_reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + "Ej5FZ+ibEtOkVkJmFBdAAA=="); + ASSERT_TRUE(data_reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { + auto unmapped_field = external_schema_field("b", 2); + unmapped_field.field_ptr->__set_name_mapping({}); + unmapped_field.field_ptr->__set_name_mapping_is_authoritative(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_TRUE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(TableReaderTest, LegacyFeEmptyNameMappingStillMatchesCurrentFileName) { + auto legacy_field = external_schema_field("b", 2); + legacy_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {legacy_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_FALSE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); @@ -4323,7 +4544,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4333,10 +4554,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { write_struct_parquet_file(file_path, 7); const auto int_type = std::make_shared(); - const auto string_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); - auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + auto missing_child = make_table_column(99, "missing_child", varbinary_type); + missing_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + missing_child.initial_default_value_is_base64 = true; + auto struct_type = std::make_shared(DataTypes {int_type, varbinary_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); struct_column.children = {id_child, missing_child}; @@ -4370,7 +4593,10 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& defaults = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(1))); + EXPECT_EQ(defaults.get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 6a986da45303d6..37dbf4cdd390ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -124,8 +124,7 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { - initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), base64InitialDefaults); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); + tSchema.setRootField(getExternalSchemaForAllColumn( + columns, nameMapping, hasNameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -161,11 +170,13 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { - return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + return getExternalSchema(columnType, dorisColumn, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); @@ -178,9 +189,12 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { - // for iceberg set name mapping. - root.setNameMapping(new ArrayList<>(nameMapping.get(dorisColumn.getUniqueId()))); + if (hasNameMapping) { + // The explicit capability keeps old-FE plans on legacy fallback while making an empty + // per-field mapping authoritative for plans produced by a compatible FE. + root.setNameMapping(new ArrayList<>( + nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); + root.setNameMappingIsAuthoritative(true); } TNestedField nestedField = new TNestedField(); @@ -199,7 +213,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( - subField.getType(), subColumn, nameMapping, base64InitialDefaults)); + subField.getType(), subColumn, nameMapping, hasNameMapping, + base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -212,7 +227,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -223,13 +238,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 05378977443f49..047272916fbee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -231,7 +231,8 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTabl icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 95c9a6f26cc5c5..a17ff87b1131c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,14 +17,30 @@ package org.apache.doris.datasource.iceberg; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; + private final Optional>> nameMapping; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { + this(partitionInfo, snapshot, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.nameMapping = nameMapping.map(mapping -> { + Map> copy = new HashMap<>(); + mapping.forEach((id, names) -> copy.put(id, List.copyOf(names))); + return Map.copyOf(copy); + }); } public IcebergPartitionInfo getPartitionInfo() { @@ -34,4 +50,8 @@ public IcebergPartitionInfo getPartitionInfo() { public IcebergSnapshot getSnapshot() { return snapshot; } + + public Optional>> getNameMapping() { + return nameMapping; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index a7908db222cd00..5022495d993994 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -109,6 +109,10 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mapping.MappedField; +import org.apache.iceberg.mapping.MappedFields; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; @@ -1089,8 +1093,18 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + String serializedDefault = serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz); + // Column constructs complex children without Iceberg field metadata. Copy through the + // public default-info API so recursive fields retain their logical pre-add value. + Column defaultCarrier = new Column(column.getName(), column.getType(), false, null, + column.isAllowNull(), serializedDefault, ""); + column.setDefaultValueInfo(defaultCarrier); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1109,7 +1123,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1166,7 +1181,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { @@ -1815,7 +1830,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( } return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId())); + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(icebergTable)); } return getLatestSnapshotCacheValue(dorisTable); } @@ -1870,6 +1886,43 @@ private static Optional loadTableSchemaCacheValue(ExternalTabl return Optional.of(new IcebergSchemaCacheValue(schema, tmpColumns)); } + /** + * Extract the Iceberg name mapping while retaining the distinction between an absent property + * and a valid empty mapping. + */ + public static Optional>> getNameMapping(Table icebergTable) { + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } + try { + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); + } + Map> result = new HashMap<>(); + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); + } catch (Exception e) { + LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); + } + } + + private static void extractMappingsFromNameMapping( + MappedFields mappingFields, Map> result) { + if (mappingFields == null) { + return; + } + for (MappedField mappedField : mappingFields.fields()) { + // Iceberg permits id-less wrapper entries; only their nested ID-bearing aliases can + // participate in Doris field-id lookup and in the immutable snapshot cache. + if (mappedField.id() != null) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + } + extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + } + } public static boolean isIcebergRowLineageColumn(Column column) { return column.nameEquals(IcebergUtils.ICEBERG_ROW_ID_COL, false) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 9ad496de40d622..d989ed0cf32ab6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -38,12 +38,15 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.profile.IcebergMetricsReporter; import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.persist.gson.GsonUtils; @@ -91,7 +94,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -100,10 +102,6 @@ import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.mapping.MappedField; -import org.apache.iceberg.mapping.MappedFields; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.ScanTaskUtil; @@ -129,6 +127,7 @@ public class IcebergScanNode extends FileQueryScanNode { public static final int MIN_DELETE_FILE_SUPPORT_VERSION = 2; + static final int ICEBERG_SCAN_SEMANTICS_VERSION = 1; private static final Logger LOG = LogManager.getLogger(IcebergScanNode.class); private IcebergSource source; @@ -264,38 +263,14 @@ protected void doInitialize() throws UserException { super.doInitialize(); } - /** - * Extract name mapping from Iceberg table properties. - * Returns a map from field ID to list of mapped names. - */ - private Map> extractNameMapping() { - Map> result = new HashMap<>(); - try { - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson != null && !nameMappingJson.isEmpty()) { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping != null) { - // Extract mappings from NameMapping - // NameMapping contains field mappings, we need to convert them to our format - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - } - } - } catch (Exception e) { - // If name mapping parsing fails, continue without it - LOG.warn("Failed to parse name mapping from Iceberg table properties", e); - } - return result; - } - - private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { - if (mappingFields == null) { - return; - } - for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); - extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + private Optional>> extractNameMapping() { + Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + // The mapping must come from the same metadata generation as the pinned schema; a + // property-only refresh can otherwise change alias semantics within one statement. + return ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getNameMapping(); } - + return IcebergUtils.getNameMapping(icebergTable); } @Override @@ -510,14 +485,23 @@ private String getDeleteFileContentType(int content) { public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); + enableCurrentIcebergScanSemantics(); // Extract name mapping from Iceberg table properties - Map> nameMapping = extractNameMapping(); + Optional>> nameMapping = extractNameMapping(); // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), - nameMapping, getBase64EncodedInitialDefaultsForScan()); + nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), + getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + void enableCurrentIcebergScanSemantics() { + // This explicit capability is the rollout boundary: old FE plans must keep legacy values + // when fragments run on a mixture of old and new BEs. + params.setIcebergScanSemanticsVersion(ICEBERG_SCAN_SEMANTICS_VERSION); } @VisibleForTesting @@ -528,17 +512,22 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } - TableScan tableScan = createTableScan(); - Snapshot snapshot = tableScan.snapshot(); - // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. - // Resolve the selected snapshot's schema id explicitly so this metadata describes the same - // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make - // BE decode a historical non-binary default as Base64, or fail to decode a binary default. - Schema scanSchema = snapshot == null - ? tableScan.schema() - : tableScan.table().schemas().get(snapshot.schemaId()); + IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = null; + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } else { + scanSchema = selectedSnapshot == null + ? icebergTable.schema() + : icebergTable.schemas().get(selectedSnapshot.getSchemaId()); + } + // A branch can expose a schema newer than its data snapshot. The statement-pinned schema + // produced the target columns, so default markers must not be recomputed from that snapshot. return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..9422f08812de2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -28,6 +28,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.schema.external.TArrayField; import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TNestedField; import org.apache.doris.thrift.schema.external.TSchema; import org.apache.doris.thrift.schema.external.TStructField; @@ -254,6 +255,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertTrue(field1.isNameMappingIsAuthoritative()); Assert.assertEquals("7", field1.getInitialDefaultValue()); Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); @@ -262,7 +264,72 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertTrue(field2.isNameMappingIsAuthoritative()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + + @Test + public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { + StructType structType = new StructType( + new StructField("added_int", Type.INT, "nested default", true)); + Column structColumn = new Column("s", structType, true); + structColumn.setUniqueId(10); + Column child = structColumn.getChildren().get(0); + child.setUniqueId(11); + child.setDefaultValueInfo(new Column("added_int", Type.INT, false, null, true, "7", "")); + TFileScanRangeParams params = new TFileScanRangeParams(); + + ExternalUtil.initSchemaInfoForAllColumn( + params, 1L, Collections.singletonList(structColumn), Collections.emptyMap()); + + TField childField = params.getHistorySchemaInfo().get(0).getRootField().getFields().get(0) + .getFieldPtr().getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals("7", childField.getInitialDefaultValue()); + Assert.assertFalse(childField.isSetInitialDefaultValueIsBase64()); + } + + @Test + public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column mappedColumn = new Column("a", Type.INT, true); + mappedColumn.setUniqueId(1); + Column unmappedColumn = new Column("b", Type.INT, true); + unmappedColumn.setUniqueId(2); + + Map> nameMapping = new HashMap<>(); + nameMapping.put(mappedColumn.getUniqueId(), Collections.singletonList("a")); + ExternalUtil.initSchemaInfoForAllColumn( + params, 600L, Arrays.asList(mappedColumn, unmappedColumn), nameMapping); + + List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); + Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(0).getFieldPtr().isNameMappingIsAuthoritative()); + Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + Assert.assertTrue(fields.get(1).getFieldPtr().isNameMappingIsAuthoritative()); + } + + @Test + public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping() { + Column column = new Column("a", Type.INT, true); + column.setUniqueId(1); + + TFileScanRangeParams absentParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn( + absentParams, 700L, Collections.singletonList(column), Collections.emptyMap()); + TField absentField = absentParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertFalse(absentField.isSetNameMapping()); + Assert.assertFalse(absentField.isSetNameMappingIsAuthoritative()); + + TFileScanRangeParams emptyParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, + Collections.singletonList(column), Collections.emptyMap(), true, Collections.emptyMap()); + TField emptyField = emptyParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertTrue(emptyField.isSetNameMapping()); + Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + Assert.assertTrue(emptyField.isNameMappingIsAuthoritative()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 0a798faff93fef..b4a1c6a5d1b235 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -188,6 +188,11 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + // The scan now resolves initial defaults from the statement-pinned schema id, so the + // mocked table must expose the same historical-schema lookup as a real Iceberg table. + Mockito.doAnswer(invocation -> ImmutableMap.of( + mockedIcebergTable.schema().schemaId(), mockedIcebergTable.schema())) + .when(mockedIcebergTable).schemas(); // Mock newScan() chain used by IcebergScanNode.createTableScan() TableScan mockedTableScan = Mockito.mock(TableScan.class, Mockito.RETURNS_DEEP_STUBS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 112873433b03ea..c9b5f13080c319 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -281,6 +281,20 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); } + @Test + public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional("added_int") + .withId(11) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build()))); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 98136848f56ca1..2742ac2aa6cd6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -19,15 +19,30 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TIcebergDeleteFileDesc; import org.apache.doris.thrift.TPushAggOp; @@ -41,6 +56,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; @@ -56,11 +72,20 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; + @SuppressWarnings("unchecked") + private static Optional>> extractNameMapping( + IcebergScanNode node) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("extractNameMapping"); + method.setAccessible(true); + return (Optional>>) method.invoke(node); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private TableScan tableScan; @@ -92,6 +117,154 @@ public boolean isBatchMode() { protected boolean getEnableMappingVarbinary() { return enableMappingVarbinary; } + + @Override + public List getPathPartitionKeys() { + return Collections.emptyList(); + } + + void addSlot(int slotId, Column column) { + SlotDescriptor slot = new SlotDescriptor(new SlotId(slotId), desc.getId()); + slot.setColumn(column); + desc.addSlot(slot); + } + + int enableAndGetIcebergScanSemanticsVersion() { + params = new TFileScanRangeParams(); + enableCurrentIcebergScanSemantics(); + return params.getIcebergScanSemanticsVersion(); + } + } + + @Test + public void testEmitsCurrentIcebergScanSemanticsCapability() { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + + Assert.assertEquals(IcebergScanNode.ICEBERG_SCAN_SEMANTICS_VERSION, + node.enableAndGetIcebergScanSemanticsVersion()); + } + + @Test + public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Table table = Mockito.mock(Table.class); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + setIcebergSource(node, source); + + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Assert.assertFalse(extractNameMapping(node).isPresent()); + + Mockito.when(table.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + Optional>> emptyMapping = extractNameMapping(node); + Assert.assertTrue(emptyMapping.isPresent()); + Assert.assertTrue(emptyMapping.get().isEmpty()); + } + + @Test + public void testSnapshotCacheIgnoresIdlessNameMappingWrapper() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap( + TableProperties.DEFAULT_NAME_MAPPING, + "[{\"names\":[\"legacy_wrapper\"],\"fields\":[" + + "{\"field-id\":7,\"names\":[\"legacy_child\"]}]}]")); + + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), + IcebergUtils.getNameMapping(table)); + + Assert.assertTrue(snapshotCacheValue.getNameMapping().isPresent()); + Assert.assertEquals(Collections.singletonList("legacy_child"), + snapshotCacheValue.getNameMapping().get().get(7)); + Assert.assertEquals(Collections.singleton(7), + snapshotCacheValue.getNameMapping().get().keySet()); + } + + @Test + public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 11L), Optional.empty()))); + try { + Assert.assertFalse(extractNameMapping(node).isPresent()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { + Schema systemTableSchema = new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.required(2, "record_count", Types.LongType.get()), + Types.NestedField.optional(3, "readable_metrics", Types.StructType.of( + Types.NestedField.optional(4, "id", Types.StructType.of( + Types.NestedField.optional(5, "lower_bound", Types.IntegerType.get())))))); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, systemTable); + node.addSlot(1, new Column("RECORD_COUNT", Type.BIGINT)); + node.addSlot(2, new Column(Column.GLOBAL_ROWID_COL + "system_table", Type.BIGINT)); + node.addSlot(3, new Column("FILE_PATH", Type.STRING)); + + List filters = Collections.singletonList(Expressions.greaterThan("record_count", 0L)); + Schema projectedSchema = node.getSystemTableProjectedSchema(filters, false); + + Assert.assertEquals(2, projectedSchema.columns().size()); + Assert.assertEquals("record_count", projectedSchema.columns().get(0).name()); + Assert.assertEquals("file_path", projectedSchema.columns().get(1).name()); + Assert.assertNull(projectedSchema.findField("readable_metrics")); + } + + @Test + public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws Exception { + Schema systemTableSchema = new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.required(2, "record_count", Types.LongType.get())); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, systemTable); + node.addSlot(1, new Column("record_count", Type.BIGINT)); + + try { + node.getSystemTableProjectedSchema( + Collections.singletonList(Expressions.equal("file_path", "data.parquet")), true); + Assert.fail("Filter columns must be materialized by the planner"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("filter column file_path is not materialized")); + } } private static class CountPlanningIcebergScanNode extends IcebergScanNode { @@ -145,7 +318,7 @@ public void testTableLevelCountSplitPlanningRequiresCountStar() { } @Test - public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") .withId(7) .ofType(Types.BinaryType.get()) @@ -160,6 +333,7 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Mockito.when(snapshot.schemaId()).thenReturn(11); Table table = Mockito.mock(Table.class); Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + Mockito.when(table.schema()).thenReturn(currentSchema); TableScan snapshotScan = Mockito.mock(TableScan.class); Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); @@ -167,11 +341,148 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(TableIf.class)); + setIcebergSource(node, source); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertTrue(defaults.isEmpty()); + } + + @Test + public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { + Schema pinnedSchema = new Schema(11, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Schema refreshedSchema = new Schema(12, + List.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); + Mockito.when(refreshedTable.schemas()).thenReturn(Map.of( + pinnedSchema.schemaId(), pinnedSchema, + refreshedSchema.schemaId(), refreshedSchema)); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, pinnedSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + IcebergTableQueryInfo selectedSnapshot = Mockito.mock(IcebergTableQueryInfo.class); + Mockito.when(selectedSnapshot.getSchemaId()).thenReturn(11); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(snapshotScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(selectedSnapshot).when(node).getSpecifiedSnapshot(); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } + @Test + public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { + Schema dataSnapshotSchema = new Schema(11, List.of(Types.NestedField.optional("string_default") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build())); + Schema branchSchema = new Schema(12, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Snapshot dataSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Map.of( + dataSnapshotSchema.schemaId(), dataSnapshotSchema, + branchSchema.schemaId(), branchSchema)); + TableScan branchScan = Mockito.mock(TableScan.class); + Mockito.when(branchScan.snapshot()).thenReturn(dataSnapshot); + Mockito.when(branchScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(branchScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, "branch", branchSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, branchSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") @@ -196,6 +507,18 @@ public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() th Mockito.verify(node, Mockito.never()).createTableScan(); } + private static void setIcebergTable(IcebergScanNode node, Table table) throws Exception { + Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); + icebergTableField.setAccessible(true); + icebergTableField.set(node, table); + } + + private static void setIcebergSource(IcebergScanNode node, IcebergSource source) throws Exception { + Field sourceField = IcebergScanNode.class.getDeclaredField("source"); + sourceField.setAccessible(true); + sourceField.set(node, source); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 46ae54781ae825..86915e46d28bfb 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -58,7 +58,10 @@ struct TField { // True when initial_default_value is Base64 and must be decoded before constructing the Doris // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. - 8: optional bool initial_default_value_is_base64 + 8: optional bool initial_default_value_is_base64, + // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the + // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. + 9: optional bool name_mapping_is_authoritative } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index b943f246dfa9eb..cf7028d996f3d7 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -512,6 +512,10 @@ struct TFileScanRangeParams { // Paimon options from FE, used for jni/native scanner // Set at ScanNode level to avoid redundant serialization in each split 30: optional map paimon_options + // Versioned Iceberg scan semantics negotiated by FE. Absence/zero preserves legacy BE + // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and + // logical initial-default materialization. + 34: optional i32 iceberg_scan_semantics_version } struct TFileRangeDesc { From 09441bbc0f696ca1a892071d25a959f11da75288 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 22 Jul 2026 20:22:57 +0800 Subject: [PATCH 15/24] [improvement](file) Build native Parquet scan path for FileScannerV2 (#65674) Issue Number: None Related PR: None FileScannerV2's Parquet path previously relied on Arrow-oriented metadata and value adapters, built intermediate decoded objects before materializing Doris columns, and lacked a single V2-owned contract for schema mapping, predicate execution, sparse reads, cache identity, and metadata aggregation. This duplicated decode/conversion work, retained large temporary buffers for nested and string columns, and made nullable sparse scans degrade into many small read/skip operations. This PR builds an independent native Parquet scan path under `be/src/format_v2`. The V2 path owns its metadata tree, page/level/value decoders, predicate scheduling, aggregate metadata handling, and Doris-column materialization. It does not call the legacy V1 Parquet reader or modify the V1 implementation under `be/src/format`. - Replace the metadata adapter with a native metadata tree shared by schema mapping, row-group planning, statistics, dictionary pruning, Bloom filters, page indexes, and scan readers. - Preserve compatible LIST/MAP/STRUCT wrapper semantics while validating malformed schema nodes, primitive-only group annotations, physical types, flat-leaf value counts, offsets, sizes, page headers, levels, dictionary IDs, and index metadata before use. - Keep schema evolution and table/file type reconciliation in the V2 `ColumnMapper` expression layer. - Decode Page V1/V2 definition and repetition levels and PLAIN, BOOLEAN RLE, dictionary, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT value streams. - Keep physical decoding in native decoders and physical-to-logical conversion in `DataTypeSerDe`, appending directly into final Doris columns. - Reconstruct ARRAY/MAP/STRUCT values from native levels and child readers; levels-only and COUNT scans avoid unused payload materialization. - Remove obsolete Arrow RecordReader/builders, metadata adapters, compatibility overloads, and I/O shims from the V2 chain. - Normalize sparse selections into selected non-null physical ranges, decode them in batches, and restore nullable logical positions separately for primitive, DECIMAL, DATE/DATETIME, string/binary, and dictionary columns. - Materialize typed dictionaries once per generation, reuse them for pruning/filtering/output, and select cache-aware direct or compact-gather paths according to dictionary and selection size. - Evaluate eligible predicate-only PLAIN `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons directly on physical values without materializing and compacting a Doris predicate column. - Retain predicate row mappings until a required multi-column/delete/output boundary and let fully filtered physical batches grow independently of final output block size. - Reuse bounded decoder, level, selection, null-map, binary, and nested scratch buffers. - Carry semantic COUNT arguments from FE only for FileScannerV2, keeping V1 file scans and internal-table planning on their existing paths. - Support exact `COUNT(nullable_col)` using definition levels while rejecting mixed `COUNT(*)` plus `COUNT(nullable_col)` pushdown when one metadata cardinality cannot represent both aggregates. - Emit synthetic metadata COUNT rows in runtime-sized batches instead of allocating a block proportional to file cardinality. - Use footer bounds for MIN/MAX aggregation only when explicit exactness flags permit it; legacy files with absent flags retain compatible behavior. - Fall back to normal scanning when aggregate metadata cannot prove an exact result. - Use V2-owned footer/page caches and stable file identities, coalesce safe remote ranges, and keep decoder/dictionary mutable state outside shared cache entries. - Add counters for native decoding, sparse selections and NULL fallback, predicate compaction, direct predicates, dictionary filtering, adaptive batching, cache activity, aggregate reads, and retained scratch. - Update the FileScannerV2 design and review documents for native metadata, decoding, selection, predicate, cache, fallback, and ownership contracts. - Unsupported encodings, conversions, nested optimization boundaries, mixed dictionary/plain chunks, incomplete indexes, or unverifiable metadata disable only the relevant optimization or return a bounded corruption error before decoder cursors are consumed. - Missing statistics, dictionaries, Bloom filters, page indexes, or stable cache identities never remove candidate rows. - All optimization fallbacks stay within V2; the legacy V1 implementation remains unchanged. FileScannerV2 Parquet scans now use a V2-owned native metadata and decoding pipeline with direct Doris-column materialization, nullable sparse decoding, cache-aware dictionaries, direct physical predicates, and bounded exact metadata aggregation. - Test: Unit Test - Review-fix BE tests: 4/4 passed. - Related BE suites (`TableReaderTest`, `NativeParquetStatisticsTest`, and `ParquetSchemaTest`): 102/102 passed under ASAN. - FE `PhysicalStorageLayerAggregateTest`: 6/6 passed with checkstyle. - Clang-format 16 and `git diff --check`: passed. - Behavior changed: Yes. FileScannerV2 uses the native V2 Parquet scan chain and exact, bounded metadata aggregate paths. - Does this need documentation: Yes. The FileScannerV2 Parquet design and review documents are updated in this PR. --- be/src/core/column/column.h | 14 + be/src/core/column/column_string.h | 103 + .../data_type_datetimev2_serde.cpp | 185 +- .../data_type_datetimev2_serde.h | 5 + .../data_type_datev2_serde.cpp | 103 +- .../data_type_serde/data_type_datev2_serde.h | 5 + .../data_type_decimal_serde.cpp | 266 +- .../data_type_serde/data_type_decimal_serde.h | 5 + .../data_type_number_serde.cpp | 339 +- .../data_type_serde/data_type_number_serde.h | 5 + .../core/data_type_serde/data_type_serde.cpp | 13 + be/src/core/data_type_serde/data_type_serde.h | 15 + .../data_type_string_serde.cpp | 114 + .../data_type_serde/data_type_string_serde.h | 5 + .../data_type_serde/data_type_time_serde.cpp | 107 + .../data_type_serde/data_type_time_serde.h | 5 + .../data_type_timestamptz_serde.cpp | 156 +- .../data_type_timestamptz_serde.h | 5 + .../data_type_varbinary_serde.cpp | 69 + .../data_type_varbinary_serde.h | 6 + .../data_type_serde/parquet_decode_source.h | 392 ++ .../core/data_type_serde/parquet_timestamp.h | 95 + be/src/exec/scan/file_scanner.h | 2 +- be/src/exec/scan/file_scanner_v2.cpp | 97 +- be/src/exec/scan/file_scanner_v2.h | 16 + be/src/exec/scan/scanner.h | 2 +- be/src/exprs/runtime_filter_expr.cpp | 235 + be/src/exprs/runtime_filter_expr.h | 146 + be/src/exprs/vcompound_pred.h | 8 + be/src/exprs/vectorized_fn_call.cpp | 151 +- be/src/exprs/vectorized_fn_call.h | 5 + be/src/exprs/vexpr.h | 23 + .../new_plain_text_line_reader.cpp | 17 +- .../file_reader/new_plain_text_line_reader.h | 6 +- .../hive_text_util.h | 0 .../jni/jni_data_bridge.cpp | 0 .../jni/jni_data_bridge.h | 0 be/src/format/table/deletion_vector.h | 25 +- .../iceberg_delete_file_reader_helper.cpp | 40 +- be/src/format/table/paimon_jni_reader.cpp | 9 +- be/src/format/table/paimon_reader.cpp | 44 +- be/src/format/table/paimon_reader.h | 7 +- be/src/format_v2/AGENTS.md | 110 + be/src/format_v2/column_mapper.cpp | 142 +- be/src/format_v2/deletion_vector.h | 41 - be/src/format_v2/deletion_vector_reader.cpp | 192 - be/src/format_v2/deletion_vector_reader.h | 111 - .../format_v2/delimited_text/csv_reader.cpp | 223 +- be/src/format_v2/delimited_text/csv_reader.h | 5 +- .../delimited_text/delimited_text_reader.cpp | 21 +- .../delimited_text/delimited_text_reader.h | 1 + .../format_v2/delimited_text/text_reader.cpp | 2 +- be/src/format_v2/expr/cast.cpp | 4 +- .../expr/equality_delete_predicate.cpp | 63 +- be/src/format_v2/file_reader.cpp | 4 + be/src/format_v2/file_reader.h | 8 + .../jni/iceberg_sys_table_reader.cpp | 43 +- be/src/format_v2/jni/jdbc_reader.cpp | 27 +- be/src/format_v2/jni/jni_table_reader.cpp | 48 +- be/src/format_v2/jni/jni_table_reader.h | 4 +- .../jni/trino_connector_jni_reader.cpp | 11 +- be/src/format_v2/json/json_reader.cpp | 37 +- be/src/format_v2/json/json_reader.h | 8 + be/src/format_v2/native/native_reader.cpp | 40 +- be/src/format_v2/native/native_reader.h | 6 + .../format_v2/orc/orc_file_input_stream.cpp | 37 +- be/src/format_v2/orc/orc_reader.cpp | 44 +- be/src/format_v2/orc/orc_reader.h | 1 + .../format_v2/parquet/native_schema_desc.cpp | 870 ++++ be/src/format_v2/parquet/native_schema_desc.h | 183 + .../format_v2/parquet/native_schema_node.cpp | 127 + be/src/format_v2/parquet/native_schema_node.h | 93 + .../parquet/parquet_column_schema.cpp | 610 +-- .../format_v2/parquet/parquet_column_schema.h | 19 +- .../parquet/parquet_file_context.cpp | 976 ++--- .../format_v2/parquet/parquet_file_context.h | 166 +- be/src/format_v2/parquet/parquet_profile.cpp | 124 +- be/src/format_v2/parquet/parquet_profile.h | 75 +- be/src/format_v2/parquet/parquet_reader.cpp | 282 +- be/src/format_v2/parquet/parquet_reader.h | 3 +- be/src/format_v2/parquet/parquet_scan.cpp | 1352 ++++-- be/src/format_v2/parquet/parquet_scan.h | 135 +- .../format_v2/parquet/parquet_statistics.cpp | 1684 ++++---- be/src/format_v2/parquet/parquet_statistics.h | 105 +- be/src/format_v2/parquet/parquet_type.cpp | 325 +- be/src/format_v2/parquet/parquet_type.h | 18 +- .../parquet/reader/column_reader.cpp | 605 +-- .../format_v2/parquet/reader/column_reader.h | 209 +- .../parquet/reader/count_column_reader.cpp | 225 + .../parquet/reader/count_column_reader.h | 72 + .../parquet/reader/list_column_reader.cpp | 230 - .../parquet/reader/list_column_reader.h | 57 - .../parquet/reader/map_column_reader.cpp | 285 -- .../parquet/reader/map_column_reader.h | 61 - .../native/block_split_bloom_filter.cpp | 116 + .../reader/native/block_split_bloom_filter.h | 54 + .../reader/native/bool_plain_decoder.cpp | 99 + .../reader/native/bool_plain_decoder.h | 107 + .../reader/native/bool_rle_decoder.cpp | 94 + .../parquet/reader/native/bool_rle_decoder.h | 61 + .../reader/native/byte_array_dict_decoder.cpp | 67 + .../reader/native/byte_array_dict_decoder.h | 45 + .../native/byte_array_plain_decoder.cpp | 131 + .../reader/native/byte_array_plain_decoder.h | 73 + .../native/byte_stream_split_decoder.cpp | 82 + .../reader/native/byte_stream_split_decoder.h | 60 + .../reader/native/column_chunk_reader.cpp | 1803 ++++++++ .../reader/native/column_chunk_reader.h | 388 ++ .../parquet/reader/native/column_reader.cpp | 1890 +++++++++ .../parquet/reader/native/column_reader.h | 656 +++ .../parquet/reader/native/common.cpp | 196 + .../format_v2/parquet/reader/native/common.h | 111 + .../parquet/reader/native/decoder.cpp | 156 + .../format_v2/parquet/reader/native/decoder.h | 345 ++ .../reader/native/delta_bit_pack_decoder.cpp | 180 + .../reader/native/delta_bit_pack_decoder.h | 812 ++++ .../reader/native/fix_length_dict_decoder.hpp | 64 + .../native/fix_length_plain_decoder.cpp | 93 + .../reader/native/fix_length_plain_decoder.h | 51 + .../parquet/reader/native/level_decoder.cpp | 271 ++ .../parquet/reader/native/level_decoder.h | 87 + .../parquet/reader/native/level_reader.cpp | 222 + .../parquet/reader/native/level_reader.h | 72 + .../parquet/reader/native/page_reader.cpp | 332 ++ .../parquet/reader/native/page_reader.h | 303 ++ .../parquet/reader/native_column_reader.cpp | 767 ++++ .../parquet/reader/native_column_reader.h | 151 + .../reader/nested_column_materializer.cpp | 70 - .../reader/nested_column_materializer.h | 45 - .../parquet/reader/parquet_leaf_reader.cpp | 803 ---- .../parquet/reader/parquet_leaf_reader.h | 173 - .../reader/row_position_column_reader.cpp | 1 + .../parquet/reader/scalar_column_reader.cpp | 584 --- .../parquet/reader/scalar_column_reader.h | 110 - .../parquet/reader/struct_column_reader.cpp | 287 -- .../parquet/reader/struct_column_reader.h | 65 - be/src/format_v2/parquet/selection_vector.h | 58 +- be/src/format_v2/table/hive_reader.cpp | 17 +- be/src/format_v2/table/hudi_reader.cpp | 35 +- be/src/format_v2/table/hudi_reader.h | 11 + ...eberg_position_delete_sys_table_reader.cpp | 16 +- ...iceberg_position_delete_sys_table_reader.h | 1 + be/src/format_v2/table/iceberg_reader.cpp | 53 +- be/src/format_v2/table/iceberg_reader.h | 4 +- be/src/format_v2/table/paimon_reader.cpp | 37 +- be/src/format_v2/table/paimon_reader.h | 13 +- .../format_v2/table/remote_doris_reader.cpp | 51 +- be/src/format_v2/table/remote_doris_reader.h | 7 + be/src/format_v2/table_reader.cpp | 118 +- be/src/format_v2/table_reader.h | 195 +- be/src/io/cache/block_file_cache_profile.cpp | 12 +- be/src/io/cache/block_file_cache_profile.h | 6 +- be/src/io/fs/buffered_reader.cpp | 11 +- be/src/io/fs/buffered_reader.h | 8 +- be/src/io/fs/hdfs_file_reader.cpp | 6 +- be/src/io/fs/hdfs_file_reader.h | 1 + be/src/io/fs/s3_file_reader.cpp | 6 +- be/src/runtime/file_scan_profile.h | 55 + be/src/runtime/runtime_state.cpp | 12 + be/src/runtime/runtime_state.h | 12 +- be/src/util/block_compression.cpp | 12 +- be/src/util/cpu_info.cpp | 11 + be/src/util/cpu_info.h | 4 + be/src/util/jdbc_utils.cpp | 67 + be/src/util/jdbc_utils.h | 48 + be/src/util/timezone_utils.cpp | 9 +- .../data_type_serde_decoded_values_test.cpp | 49 +- .../data_type_serde_parquet_test.cpp | 891 ++++ be/test/exec/scan/file_scanner_v2_test.cpp | 20 + be/test/format_v2/column_mapper_test.cpp | 155 +- .../delimited_text/csv_reader_test.cpp | 16 +- .../delimited_text/text_reader_test.cpp | 44 +- .../format_v2/jni/jni_table_reader_test.cpp | 45 +- .../orc/orc_file_input_stream_test.cpp | 47 +- be/test/format_v2/orc/orc_reader_test.cpp | 183 +- .../format_v2/parquet/native_decoder_test.cpp | 3773 +++++++++++++++++ .../parquet_benchmark_scenarios_test.cpp | 148 + .../parquet/parquet_column_reader_test.cpp | 3682 ---------------- .../parquet/parquet_leaf_reader_test.cpp | 506 --- .../parquet/parquet_page_cache_range_test.cpp | 141 +- .../parquet/parquet_reader_control_test.cpp | 1146 +---- .../format_v2/parquet/parquet_reader_test.cpp | 1238 ++++-- .../format_v2/parquet/parquet_scan_test.cpp | 1085 ++++- .../format_v2/parquet/parquet_schema_test.cpp | 1010 +++-- .../parquet/parquet_serde_reader_test.cpp | 459 -- .../parquet/parquet_statistics_test.cpp | 1445 +++---- .../format_v2/parquet/parquet_type_test.cpp | 491 +-- be/test/format_v2/table/hive_reader_test.cpp | 2 +- be/test/format_v2/table/hudi_reader_test.cpp | 61 + ..._position_delete_sys_table_reader_test.cpp | 63 + .../format_v2/table/iceberg_reader_test.cpp | 224 +- .../format_v2/table/paimon_reader_test.cpp | 64 +- be/test/format_v2/table_reader_test.cpp | 181 +- docs/file-scanner-v2-code-review-guide.md | 150 +- docs/file-scanner-v2-design.md | 17 + docs/file-scanner-v2-parquet-scan-design.md | 445 +- .../doris/common/profile/SummaryProfile.java | 4 - .../doris/datasource/FileQueryScanNode.java | 44 +- .../iceberg/IcebergSnapshotCacheValue.java | 8 +- .../iceberg/source/IcebergScanNode.java | 2 + .../translator/PhysicalPlanTranslator.java | 1 - .../implementation/AggregateStrategies.java | 24 +- .../PhysicalStorageLayerAggregate.java | 17 +- .../org/apache/doris/planner/PlanNode.java | 1 + .../datasource/FileQueryScanNodeTest.java | 76 +- .../helper/IcebergWriterHelperTest.java | 85 +- .../iceberg/source/IcebergScanNodeTest.java | 97 +- .../paimon/source/PaimonScanNodeTest.java | 10 +- .../tvf/source/TVFScanNodeTest.java | 3 +- .../postprocess/TopnLazyMaterializeTest.java | 35 +- .../PhysicalStorageLayerAggregateTest.java | 80 + .../doris/planner/IcebergMergeSinkTest.java | 9 +- .../statistics/util/StatisticsUtilTest.java | 24 +- gensrc/thrift/PlanNodes.thrift | 14 + .../tvf/test_hdfs_parquet_group6.out | 16 +- .../hive/test_orc_lazy_mat_profile.groovy | 32 +- 216 files changed, 28178 insertions(+), 16127 deletions(-) create mode 100644 be/src/core/data_type_serde/parquet_decode_source.h create mode 100644 be/src/core/data_type_serde/parquet_timestamp.h create mode 100644 be/src/exprs/runtime_filter_expr.cpp create mode 100644 be/src/exprs/runtime_filter_expr.h rename be/src/{format_v2/delimited_text => format}/hive_text_util.h (100%) rename be/src/{format_v2 => format}/jni/jni_data_bridge.cpp (100%) rename be/src/{format_v2 => format}/jni/jni_data_bridge.h (100%) delete mode 100644 be/src/format_v2/deletion_vector.h delete mode 100644 be/src/format_v2/deletion_vector_reader.cpp delete mode 100644 be/src/format_v2/deletion_vector_reader.h create mode 100644 be/src/format_v2/parquet/native_schema_desc.cpp create mode 100644 be/src/format_v2/parquet/native_schema_desc.h create mode 100644 be/src/format_v2/parquet/native_schema_node.cpp create mode 100644 be/src/format_v2/parquet/native_schema_node.h create mode 100644 be/src/format_v2/parquet/reader/count_column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/count_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/list_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/list_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/map_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/map_column_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp create mode 100644 be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h create mode 100644 be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/bool_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/bool_rle_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/column_chunk_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/column_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/common.cpp create mode 100644 be/src/format_v2/parquet/reader/native/common.h create mode 100644 be/src/format_v2/parquet/reader/native/decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/level_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/level_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/level_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/level_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/page_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/page_reader.h create mode 100644 be/src/format_v2/parquet/reader/native_column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/nested_column_materializer.cpp delete mode 100644 be/src/format_v2/parquet/reader/nested_column_materializer.h delete mode 100644 be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/parquet_leaf_reader.h delete mode 100644 be/src/format_v2/parquet/reader/scalar_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/scalar_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/struct_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/struct_column_reader.h create mode 100644 be/src/runtime/file_scan_profile.h create mode 100644 be/src/util/jdbc_utils.cpp create mode 100644 be/src/util/jdbc_utils.h create mode 100644 be/test/core/data_type_serde/data_type_serde_parquet_test.cpp create mode 100644 be/test/format_v2/parquet/native_decoder_test.cpp create mode 100644 be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp delete mode 100644 be/test/format_v2/parquet/parquet_column_reader_test.cpp delete mode 100644 be/test/format_v2/parquet/parquet_leaf_reader_test.cpp delete mode 100644 be/test/format_v2/parquet/parquet_serde_reader_test.cpp create mode 100644 be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index 9547ff1a7ca3da..3b727285e64613 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -262,6 +262,14 @@ class IColumn : public COW { "Method insert_many_continuous_binary_data is not supported for " + get_name()); } + // OFFSET_ONLY readers still need true string lengths; keeping this virtual preserves that + // invariant when decoders only hold an IColumn reference and intentionally skip payloads. + virtual void insert_offsets_from_lengths(const uint32_t* lengths, size_t num) { + throw doris::Exception( + ErrorCode::NOT_IMPLEMENTED_ERROR, + "Method insert_offsets_from_lengths is not supported for " + get_name()); + } + virtual void insert_many_strings(const StringRef* strings, size_t num) { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "Method insert_many_strings is not supported for " + get_name()); @@ -682,6 +690,12 @@ class IColumn : public COW { */ String dump_structure() const; + Status column_self_check() const { + // branch-4.1 predates the recursive debug validators used by master format_v2; keeping + // this check side-effect free preserves the branch's established column semantics. + return Status::OK(); + } + // only used in agg value replace for column which is not variable length, eg.BlockReader::_copy_value_data // usage: self_column.replace_column_data(other_column, other_column's row index, self_column's row index) virtual void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) = 0; diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 8972a4ca210ceb..7b6317e297f18c 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -277,6 +278,79 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + template + void insert_many_parquet_plain_byte_arrays(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num, + const std::vector& value_spans) { + if (UNLIKELY(num == 0)) { + return; + } + const size_t old_chars_size = chars.size(); + const size_t bytes = value_offsets[num]; + check_chars_length(old_chars_size + bytes, offsets.size() + num); + chars.resize(old_chars_size + bytes); + + // Source payloads may be contiguous for decoder-owned buffers even though PLAIN normally + // separates them with length prefixes. Coalesce whenever the published layout permits it; + // the fallback remains one direct source-to-column copy without a StringRef staging array. + size_t covered_values = 0; + for (const auto& span : value_spans) { + DORIS_CHECK_EQ(span.first, covered_values); + DORIS_CHECK_LE(span.first + span.count, num); + size_t run_first = span.first; + const size_t span_end = span.first + span.count; + while (run_first < span_end) { + size_t run_end = run_first + 1; + while (run_end < span_end && + payload_offsets[run_end] == payload_offsets[run_first] + + value_offsets[run_end] - + value_offsets[run_first]) { + ++run_end; + } + const size_t run_bytes = value_offsets[run_end] - value_offsets[run_first]; + if (run_bytes != 0) { + memcpy(chars.data() + old_chars_size + value_offsets[run_first], + encoded_data + payload_offsets[run_first], run_bytes); + } + run_first = run_end; + } + covered_values = span_end; + } + DORIS_CHECK_EQ(covered_values, num); + + const size_t old_rows = offsets.size(); + const auto tail_offset = offsets.back(); + offsets.resize(old_rows + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_rows + row] = tail_offset + value_offsets[row + 1]; + } + sanity_check_simple(); + } + + // Insert `num` string entries with real length information but no actual + // character data. The `lengths` array provides the byte length of each + // string. Offsets are built with correct cumulative sizes so that + // size_at(i) returns the true string length. The chars buffer is extended + // with zero-filled padding to maintain the invariant chars.size() == offsets.back(). + // Used by OFFSET_ONLY reading mode where actual string content is not needed + // but length information must be preserved (e.g., for length() function). + void insert_offsets_from_lengths(const uint32_t* lengths, size_t num) override { + if (UNLIKELY(num == 0)) { + return; + } + const auto old_rows = offsets.size(); + // Build cumulative offsets from lengths + offsets.resize(old_rows + num); + auto* offsets_ptr = &offsets[old_rows]; + size_t running_offset = offsets[old_rows - 1]; + for (size_t i = 0; i < num; ++i) { + running_offset += lengths[i]; + offsets_ptr[i] = static_cast(running_offset); + } + chars.resize(offsets[old_rows + num - 1]); + } + void insert_many_strings(const StringRef* strings, size_t num) override { size_t new_size = 0; for (size_t i = 0; i < num; i++) { @@ -300,6 +374,35 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + void insert_many_fixed_length_data(const char* data, size_t value_length, size_t num) { + if (num == 0) { + return; + } + if (value_length > std::numeric_limits::max() / num) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + const size_t old_chars_size = chars.size(); + const size_t old_offsets_size = offsets.size(); + const size_t bytes = value_length * num; + if (bytes > std::numeric_limits::max() - old_chars_size || + num > std::numeric_limits::max() - old_offsets_size) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + check_chars_length(old_chars_size + bytes, old_offsets_size + num); + chars.resize(old_chars_size + bytes); + if (bytes != 0) { + memcpy(chars.data() + old_chars_size, data, bytes); + } + offsets.resize(old_offsets_size + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_offsets_size + row] = + static_cast(old_chars_size + (row + 1) * value_length); + } + sanity_check_simple(); + } + template void insert_many_strings_fixed_length(const StringRef* strings, size_t num) { size_t new_size = 0; diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 5b372cce23bd65..787c73a8fa1a30 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,10 +31,13 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "util/unaligned.h" enum { DIVISOR_FOR_SECOND = 1, @@ -49,22 +52,6 @@ static const int64_t micro_to_nano_second = 1000; namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -106,9 +93,9 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, return Status::OK(); } -void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, - int64_t timestamp_micros, - const cctz::time_zone& timezone) { +Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { static constexpr int64_t MICROS_PER_SECOND = 1000000; int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; @@ -121,19 +108,91 @@ void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + if (!datetime_value.is_valid_date()) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", + timestamp_micros); + } data.push_back(datetime_value); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + status = _context.physical_type == ParquetPhysicalType::INT96 || + _context.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(_data, timestamp_micros); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATETIMEV2"); + } +}; + } // namespace // NOLINTBEGIN(readability-function-size) @@ -561,7 +620,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); static const auto utc_timezone = cctz::utc_time_zone(); const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; for (int64_t row = 0; row < view.row_count; ++row) { @@ -569,8 +628,19 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), - timezone); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -583,24 +653,63 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); - if (view.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); - } else { - auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); - if (!st.ok()) { - if (decoded_column_view_can_null_on_conversion_failure(view)) { - decoded_column_view_insert_null_on_conversion_failure(column, view, row); - continue; - } - data.resize(old_size); - return st; + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = view.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; } + data.resize(old_size); + return status; } } return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateTimeV2ParquetConsumer consumer(column, context); + RejectDateTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); + } + DateTimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateTimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectDateTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index e089d789ccee7d..c3411ed4249843 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -90,6 +90,11 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe* value) { + DORIS_CHECK(value != nullptr); + const int64_t day_number = static_cast(encoded_date) + date_threshold; + if (day_number < 0 || !value->get_date_from_daynr(static_cast(day_number))) { + return Status::DataQualityError("Parquet DATE value is out of range"); + } + return Status::OK(); +} + +class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + explicit DateV2ParquetConsumer(IColumn& column, ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + DateV2Value value; + const auto status = decode_parquet_date( + unaligned_load(values + row * sizeof(int32_t)), &value); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = DateV2Value(); + continue; + } + _data.resize(old_size); + return status; + } + _data[old_size + row] = value; + } + return Status::OK(); + } + +private: + ColumnDateV2::Container& _data; + ParquetMaterializationState* _state; +}; + +class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATEV2"); + } +}; + +} // namespace Status DataTypeDateV2SerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, @@ -142,6 +194,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { @@ -149,12 +202,56 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, continue; } DateV2Value date_v2; - date_v2.get_date_from_daynr(values[row] + date_threshold); + const auto status = decode_parquet_date(values[row], &date_v2); + if (!status.ok()) { + // Decoded values back both metadata conversion and native materialization. Preserve + // strict errors while allowing nullable non-strict scans to mark only the bad row. + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } data.push_back(date_v2); } return Status::OK(); } +Status DataTypeDateV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateV2ParquetConsumer consumer(column); + RejectDateV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT32 || + context.logical_type != ParquetLogicalType::DATE) { + return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); + } + DateV2ParquetConsumer consumer(column, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, &state); + RejectDateV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index bc02b61b520193..39d44efcfa0652 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -88,6 +88,11 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe #include +#include +#include #include #include "arrow/type.h" @@ -35,6 +37,7 @@ #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" #include "exprs/function/cast/cast_to_decimal.h" @@ -74,8 +77,13 @@ NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { template bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, UInt32 precision) { - return value >= min_decimal_value(precision).value && - value <= max_decimal_value(precision).value; + if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; + } else { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; + } } template @@ -84,6 +92,9 @@ bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { if constexpr (std::is_same_v) { const auto wide_value = wide::Int256(value); return decoded_decimal_value_fits(wide_value, precision); + } else if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; } else { return value >= static_cast(min_decimal_value(precision).value) && value <= static_cast(max_decimal_value(precision).value); @@ -178,6 +189,214 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie return Status::OK(); } +template +wide::Int256 parquet_decimal_limit(UInt32 precision) { + if constexpr (T == TYPE_DECIMALV2) { + return wide::Int256(DataTypeDecimal::get_max_digits_number(precision)); + } else { + return wide::Int256(max_decimal_value(precision).value); + } +} + +template +Status scale_parquet_decimal(wide::Int256 value, int32_t source_scale, int32_t target_scale, + UInt32 target_precision, wide::Int256* result) { + DORIS_CHECK(result != nullptr); + const auto limit = parquet_decimal_limit(target_precision); + if (source_scale > target_scale) { + const int64_t scale_delta = static_cast(source_scale) - target_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // The precision bound above guarantees that narrowing the positive scale delta keeps + // the multiplier lookup in range. + const auto divisor = + decimal_scale_multiplier(static_cast(scale_delta)); + // Scale-down must be exact. Truncating before the target precision check silently + // changes source values and makes plain and dictionary decoding disagree with casts. + if (value % divisor != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + value /= divisor; + } + } else if (source_scale < target_scale) { + const int64_t scale_delta = static_cast(target_scale) - source_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // Keep the same checked narrowing invariant for scale-up as for exact scale-down. + const auto multiplier = + decimal_scale_multiplier(static_cast(scale_delta)); + if (value > limit / multiplier || value < -limit / multiplier) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + value *= multiplier; + } + } + if (value < -limit || value > limit) { + return Status::DataQualityError("Parquet decimal value is out of range"); + } + *result = value; + return Status::OK(); +} + +template +class DecimalParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + + DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + UInt32 target_precision, int32_t target_scale, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast&>(column).get_data()), + _context(context), + _target_precision(target_precision), + _target_scale(target_scale), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_integers(values, num_values); + } + if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_integers(values, num_values); + } + if (_context.physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return Status::NotSupported("Unsupported Parquet physical type {} for decimal SerDe", + static_cast(_context.physical_type)); + } + DORIS_CHECK_EQ(value_width, static_cast(_context.type_length)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = + append_binary_value(values + row * value_width, value_width, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = append_binary_value(reinterpret_cast(values[row].data), + values[row].size, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const size_t length = value_offsets[row + 1] - value_offsets[row]; + auto status = append_binary_value( + reinterpret_cast(encoded_data + payload_offsets[row]), length, + old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + template + Status append_integers(const uint8_t* values, size_t num_values) { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + constexpr int32_t SOURCE_DIGITS = std::numeric_limits::digits10 + 1; + if (_context.decimal_scale == _target_scale && + SOURCE_DIGITS <= static_cast(_target_precision)) { + // The complete physical domain fits the target at the same scale, so narrowing cannot + // precede a failure check and the hot same-scale path needs no wide arithmetic. + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = + unaligned_load(values + row * sizeof(SourceType)); + _data[old_size + row] = FieldType {NativeType(source_value)}; + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = unaligned_load(values + row * sizeof(SourceType)); + auto status = append_wide_value(wide::Int256(source_value), old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { + if (UNLIKELY(length > sizeof(wide::Int256))) { + return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); + } + return append_wide_value( + decode_big_endian_signed_integer(value, cast_set(length)), + output_row); + } + + Status append_wide_value(wide::Int256 value, size_t output_row) { + wide::Int256 scaled_value; + RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, + _target_precision, &scaled_value)); + // Narrow only after scaling and target-precision validation. In particular, an INT64 or a + // sign-extended binary value must never wrap through Decimal32 before it can be rejected. + _data[output_row] = FieldType {static_cast(scaled_value)}; + return Status::OK(); + } + + typename ColumnDecimal::Container& _data; + const ParquetDecodeContext& _context; + UInt32 _target_precision; + int32_t _target_scale; + ParquetMaterializationState* _state; +}; + } // namespace template @@ -529,6 +748,49 @@ Status DataTypeDecimalSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeDecimalSerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DecimalParquetConsumer dictionary_consumer(*state.typed_dictionary, context, + cast_set(precision), scale, &state); + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, dictionary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 9f8fd58799b753..bc79b22c417ea0 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -110,6 +110,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { int64_t end, const cctz::time_zone& ctz) const override; Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index af6ae9d7f43a72..abd3b082df74bf 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -50,6 +53,23 @@ namespace doris { namespace { +float parquet_half_to_float(uint16_t half) { + const uint32_t sign = (half & 0x8000U) << 16; + const uint32_t exponent = (half & 0x7C00U) >> 10; + const uint32_t mantissa = half & 0x03FFU; + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float value = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? value : -value; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + template const NativeType* decoded_values_as(const DecodedColumnView& view) { return reinterpret_cast(view.values); @@ -77,6 +97,43 @@ bool decoded_number_value_fits(SourceType value) { } } +template +constexpr bool parquet_number_conversion_always_fits() { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (!std::is_integral_v || !std::is_integral_v || + std::is_same_v) { + return false; + } else if constexpr (std::is_signed_v == std::is_signed_v) { + return sizeof(DorisCppType) >= sizeof(SourceType); + } else if constexpr (std::is_signed_v) { + return sizeof(DorisCppType) > sizeof(SourceType); + } else { + return false; + } +} + +template +bool parquet_logical_integer_carrier_fits(SourceType value) { + if constexpr (sizeof(LogicalType) < sizeof(SourceType)) { + // Parquet INT(bitWidth) annotations constrain the physical INT32/INT64 carrier. Validate + // before narrowing so malformed values cannot wrap into an apparently valid logical value. + if constexpr (std::is_signed_v) { + const auto widened = static_cast(value); + if constexpr (std::is_signed_v) { + return widened >= static_cast(std::numeric_limits::lowest()) && + widened <= static_cast(std::numeric_limits::max()); + } + return widened >= 0 && + static_cast(widened) <= + static_cast(std::numeric_limits::max()); + } + const auto widened = static_cast(value); + return widened <= static_cast(std::numeric_limits::max()); + } + return true; +} + template Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -121,7 +178,21 @@ Status read_logical_integer_decoded_values_as(IColumn& column, const DecodedColu data.push_back(DorisCppType()); continue; } - const auto logical_value = static_cast(values[row]); + const auto physical_value = values[row]; + // Predicate decoding must match permissive materialization: annotated narrow integers use + // their declared bit width unless strict metadata validation explicitly requests rejection. + if (view.enable_strict_mode && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Decoded logical integer carrier is out of range for {} at row {}", + column.get_name(), row); + } + const auto logical_value = static_cast(physical_value); if (!decoded_number_value_fits(logical_value)) { if (decoded_column_view_can_null_on_conversion_failure(view)) { decoded_column_view_insert_null_on_conversion_failure(column, view, row); @@ -178,6 +249,226 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie } } +template +Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (std::is_same_v) { + // Identical fixed-width physical/logical types need no validation or conversion. Parquet + // PLAIN values and Doris POD columns share the byte representation on supported targets, + // so preserve one dense vector-at-a-time memcpy instead of converting value by value. + memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); + return Status::OK(); + } + if constexpr (parquet_number_conversion_always_fits()) { + // A widening conversion cannot fail, so keeping range checks in the row loop only blocks + // auto-vectorization. Input can be unaligned at a Parquet page boundary; load explicitly. + for (size_t row = 0; row < num_values; ++row) { + data[old_size + row] = static_cast( + unaligned_load(values + row * sizeof(SourceType))); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load(values + row * sizeof(SourceType)); + if (!decoded_number_value_fits(value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet value is out of range at row {}", row); + } + data[old_size + row] = static_cast(value); + } + return Status::OK(); +} + +template +Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (parquet_number_conversion_always_fits() && + sizeof(LogicalType) >= sizeof(SourceType)) { + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = + unaligned_load(values + row * sizeof(SourceType)); + data[old_size + row] = + static_cast(static_cast(physical_value)); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); + // Permissive scans preserve the long-standing bit-width interpretation of annotated + // carriers; strict scans still reject malformed physical values before narrowing. + if ((state == nullptr || state->enable_strict_mode) && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Parquet logical integer carrier is out of range at row {}", row); + } + const auto logical_value = static_cast(physical_value); + if (!decoded_number_value_fits(logical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet logical integer is out of range at row {}", + row); + } + data[old_size + row] = static_cast(logical_value); + } + return Status::OK(); +} + +template +Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + if (context.logical_integer_bit_width <= 0) { + return append_parquet_number(data, values, num_values, context, + state); + } + if (context.logical_integer_is_signed) { + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers( + data, values, num_values, state); + case 16: + return append_parquet_logical_integers( + data, values, num_values, state); + case 32: + return append_parquet_logical_integers( + data, values, num_values, state); + case 64: + return append_parquet_logical_integers( + data, values, num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } + } + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values, state); + case 16: + return append_parquet_logical_integers(data, values, + num_values, state); + case 32: + return append_parquet_logical_integers(data, values, + num_values, state); + case 64: + return append_parquet_logical_integers(data, values, + num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } +} + +template +class NumberParquetConsumer final : public ParquetFixedValueConsumer { +public: + using DorisCppType = typename PrimitiveTypeTraits::CppType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return consume_impl(values, num_values, value_width); + } + + Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) override { + // Each selected PLAIN range is already contiguous in the encoded page. Append those spans + // directly so sparse reads never build an intermediate selected-values array. + for (const auto& range : ranges) { + RETURN_IF_ERROR( + consume_impl(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } + +private: + Status consume_impl(const uint8_t* values, size_t num_values, size_t value_width) { + if (_context.logical_float16) { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); + DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const float value = parquet_half_to_float( + unaligned_load(values + row * sizeof(uint16_t))); + if (!decoded_number_value_fits(value)) { + if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { + _data[old_size + row] = DorisCppType(); + DORIS_CHECK(_state->mark_conversion_failure(old_size + row)); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet FLOAT16 value is out of range at row {}", row); + } + _data[old_size + row] = static_cast(value); + } + return Status::OK(); + } + switch (_context.physical_type) { + case ParquetPhysicalType::BOOLEAN: + DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::INT32: + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::INT64: + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::FLOAT: + DORIS_CHECK_EQ(value_width, sizeof(float)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::DOUBLE: + DORIS_CHECK_EQ(value_width, sizeof(double)); + return append_parquet_number(_data, values, num_values, _context, + _state); + default: + return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", + static_cast(_context.physical_type)); + } + } + + PaddedPODArray& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as a number"); + } +}; + } // namespace // Basic structure of the type map. template @@ -329,6 +620,52 @@ Status DataTypeNumberSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeNumberSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_parquet_dictionary(column, source, context); + } else { + NumberParquetConsumer consumer(column, context); + RejectParquetBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); + } +} + +template +Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); + } else { + NumberParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + NumberParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectParquetBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); + } +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 16c4ccd48e5c8a..f2cf7d26200f34 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -120,6 +120,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index da5db282337883..47e2c2ab983ae8 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -52,6 +52,7 @@ #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/data_type_timestamptz_serde.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -710,6 +711,18 @@ Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, get_name())); } +Status DataTypeSerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + return Status::NotSupported("read_column_from_parquet is not supported for {}", get_name()); +} + +Status DataTypeSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + return Status::NotSupported("read_parquet_dictionary is not supported for {}", get_name()); +} + Status DataTypeSerDe::read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const { return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 2c32504984a237..70ae8e4758b4fa 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -94,6 +94,9 @@ struct CastParameters; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; +class ParquetDecodeSource; +struct ParquetDecodeContext; +struct ParquetMaterializationState; /// Info that represents a scalar or array field in a decomposed view. /// It allows to recreate field with different number @@ -503,6 +506,18 @@ class DataTypeSerDe { // the Doris-type-specific materialization into IColumn. virtual Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const; + + // Read encoded Parquet values directly into the destination Doris column. ColumnReader owns + // levels/null/filter handling; the source owns only encoding-stream state; the target SerDe + // owns all physical/logical type interpretation and materialization. + virtual Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const; + // Decode one dictionary page into the selected Doris type without consuming data-page + // indices. Dictionary filters use this to keep type interpretation in SerDe instead of + // exposing dictionary bytes or decoder-owned strings to ColumnReader. + virtual Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const; virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const; diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 80cf74634f0a16..1461c1d8746470 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -17,14 +17,18 @@ #include "core/data_type_serde/data_type_string_serde.h" +#include #include #include +#include #include "common/config.h" #include "core/column/column_string.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -57,6 +61,65 @@ Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view return Status::OK(); } +template +class StringParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_fixed_length_data(static_cast(nullptr), + size_t(), size_t()); + }) { + // FIXED_LEN_BYTE_ARRAY is already a dense byte span. Copy it once and synthesize + // offsets; StringRef batches add a second row loop and hundreds of tiny memcpy calls. + _column.insert_many_fixed_length_data(reinterpret_cast(values), + value_width, num_values); + } else { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + _column.insert_many_strings(values, num_values); + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_parquet_plain_byte_arrays( + encoded_data, payload_offsets, value_offsets, num_values, + value_spans); + }) { + _column.insert_many_parquet_plain_byte_arrays(encoded_data, payload_offsets, + value_offsets, num_values, value_spans); + return Status::OK(); + } + return ParquetBinaryValueConsumer::consume_plain_byte_array( + encoded_data, payload_offsets, value_offsets, num_values, value_spans); + } + +private: + ColumnType& _column; +}; + } // namespace namespace { @@ -497,6 +560,57 @@ Status DataTypeStringSerDeBase::read_column_from_decoded_values( return read_string_decoded_values(column, view); } +template +Status DataTypeStringSerDeBase::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + StringParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeStringSerDeBase::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + auto& indices = assert_cast(column).get_data(); + const size_t old_size = indices.size(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + StringParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type {} for string SerDe", + static_cast(context.physical_type)); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index 6c64b1276c3afb..6389b6be92546e 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -206,6 +206,11 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c40e671793c848..52d387eda5ca6d 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -17,13 +17,17 @@ #include "core/data_type_serde/data_type_time_serde.h" +#include + #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" +#include "util/unaligned.h" namespace doris { namespace { @@ -51,6 +55,74 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); } +class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + int64_t raw_value; + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + raw_value = unaligned_load(values + row * sizeof(int32_t)); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + raw_value = unaligned_load(values + row * sizeof(int64_t)); + } + + int64_t units_per_day; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + units_per_day = 86400000; + } else if (_context.time_unit == ParquetTimeUnit::MICROS) { + units_per_day = 86400000000; + } else { + DORIS_CHECK(_context.time_unit == ParquetTimeUnit::NANOS); + units_per_day = 86400000000000; + } + // Validate the declared carrier before rescaling: truncating nanoseconds first could + // turn a value at or beyond 24:00:00 into an apparently valid TIMEV2 value. + if (raw_value < 0 || raw_value >= units_per_day) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value {} is outside the one-day domain", raw_value); + } + int64_t micros = raw_value; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; + } + // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields + // and immediately recombining them is an identity operation with several divisions. + _data[old_size + row] = static_cast(micros); + } + return Status::OK(); + } + +private: + ColumnTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMEV2"); + } +}; + } // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, @@ -193,6 +265,41 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + TimeV2ParquetConsumer consumer(column, context); + RejectTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if ((context.physical_type != ParquetPhysicalType::INT32 && + context.physical_type != ParquetPhysicalType::INT64) || + context.logical_type != ParquetLogicalType::TIME) { + return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); + } + TimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index c3b8ab41a0fdd5..c7f3e15383f54f 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -70,6 +70,11 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe(micros_of_second)); + if (!timestamp_tz.is_valid_date()) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } data.push_back(timestamp_tz); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnTimeStampTz::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMPTZ"); + } +}; + } // namespace // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp @@ -308,14 +355,27 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -326,11 +386,57 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampTzParquetConsumer consumer(column, context); + RejectTimestampTzBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); + } + TimestampTzParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampTzParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimestampTzBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 23d57f57fc8dac..6777459909a090 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -75,6 +75,11 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(reinterpret_cast(values + row * value_width), + value_width); + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(values[row].data, values[row].size); + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return Status::OK(); + } + +private: + ColumnVarbinary& _column; +}; + +} // namespace + +Status DataTypeVarbinarySerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + VarbinaryParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + VarbinaryParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type for VARBINARY"); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& arena, int32_t col_id, int64_t row_num, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index a3ce30f6f7dac8..cf4d33a99de11e 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -82,6 +82,12 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { "read_column_from_arrow with type " + column.get_name()); } + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h new file mode 100644 index 00000000000000..e9762385689d07 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +// These enums deliberately do not expose parquet thrift classes to the core type system. The +// format reader translates the thrift metadata once when it creates a column reader. +enum class ParquetPhysicalType { + BOOLEAN, + INT32, + INT64, + INT96, + FLOAT, + DOUBLE, + BYTE_ARRAY, + FIXED_LEN_BYTE_ARRAY, +}; + +enum class ParquetValueEncoding { + PLAIN, + DICTIONARY, + RLE, + BIT_PACKED, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, + BYTE_STREAM_SPLIT, +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +enum class ParquetLogicalType { + NONE, + STRING, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTEGER, + UUID, + FLOAT16, +}; + +// Immutable metadata required to turn one Parquet physical value into the selected Doris type. +// Encoding describes how the value source is read; logical annotations describe its meaning. +struct ParquetDecodeContext { + ParquetPhysicalType physical_type = ParquetPhysicalType::INT32; + ParquetValueEncoding encoding = ParquetValueEncoding::PLAIN; + ParquetLogicalType logical_type = ParquetLogicalType::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + + int32_t type_length = -1; + int32_t decimal_precision = -1; + int32_t decimal_scale = -1; + int32_t logical_integer_bit_width = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + bool logical_float16 = false; + bool logical_uuid = false; + bool dictionary_index_only = false; + + const cctz::time_zone* timezone = nullptr; +}; + +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + +// A decoder may produce multiple contiguous spans for one request (for example delta encodings). +// Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. +class ParquetFixedValueConsumer { +public: + virtual ~ParquetFixedValueConsumer() = default; + virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; + virtual Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) { + for (const auto& range : ranges) { + RETURN_IF_ERROR(consume(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } +}; + +class ParquetBinaryValueConsumer { +public: + virtual ~ParquetBinaryValueConsumer() = default; + virtual Status consume(const StringRef* values, size_t num_values) = 0; + + // PLAIN BYTE_ARRAY decoders already have to parse every length prefix. Publish the parsed + // source and destination offsets so string columns do not rebuild an equally large StringRef + // array and rescan all lengths before copying. Spans are expressed in output coordinates and + // preserve adjacent surviving runs for consumers that can amortize range setup. + virtual Status consume_plain_byte_array(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) { + std::vector values; + values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + values.emplace_back(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return consume(values.data(), values.size()); + } +}; + +// Dictionary decoders publish validated IDs without knowing the destination Doris type. A +// cache-resident dictionary can therefore fuse RLE decode with target-column gathering, while a +// large sparse dictionary can still choose a compact ID buffer before random dictionary access. +class ParquetDictionaryValueConsumer { +public: + virtual ~ParquetDictionaryValueConsumer() = default; + virtual Status consume_indices(const uint32_t* indices, size_t num_values) = 0; + virtual Status consume_repeated(uint32_t index, size_t num_values) { + std::vector indices(num_values, index); + return consume_indices(indices.data(), indices.size()); + } +}; + +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs do +// not occupy the encoded value stream, so the native ColumnReader merges the logical selection +// with definition levels before constructing this plan. The SerDe sees only selected non-NULL +// payload, while the reader restores selected NULL slots after the compact decode. Ranges are +// sorted, disjoint, and expressed in the physical value stream's coordinate space. +struct ParquetSelection { + size_t total_values = 0; + size_t selected_values = 0; + std::vector ranges; +}; + +// Encoding decoders implement this interface. They own encoded-stream cursors and dictionary +// storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer +// and therefore the physical/logical-to-Doris conversion. +class ParquetDecodeSource { +public: + virtual ~ParquetDecodeSource() = default; + + virtual Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) = 0; + virtual Status decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) = 0; + virtual Status skip_values(size_t num_values) = 0; + + // Batch-level sparse decode. The default implementation preserves every encoding's cursor + // semantics while moving SerDe dispatch and consumer construction out of the selection-run + // loop. Decoders with cheap random access or batch decode override these methods to remove the + // remaining per-range virtual calls as well. + virtual Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_fixed_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_binary_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + + virtual bool has_dictionary() const { return false; } + virtual uint64_t dictionary_generation() const { return 0; } + virtual size_t dictionary_size() const { return 0; } + virtual Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return Status::NotSupported("Parquet dictionary is not supported by this decoder"); + } + virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { + return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); + } + virtual Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) { + DORIS_CHECK(indices != nullptr); + indices->clear(); + indices->reserve(selection.selected_values); + std::vector range_indices; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_dictionary_indices(range.count, &range_indices)); + indices->insert(indices->end(), range_indices.begin(), range_indices.end()); + cursor = range.first + range.count; + } + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(indices->size(), selection.selected_values); + return Status::OK(); + } + virtual Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const { + return false; + } +}; + +enum class ParquetDictionaryMaterializationStrategy : uint8_t { DIRECT, INDICES }; + +// Dictionary values are materialized once into the selected Doris type. The state belongs to a +// column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. +struct ParquetMaterializationState { + MutableColumnPtr typed_dictionary; + std::vector dictionary_indices; + ParquetSelection selection; + uint64_t dictionary_generation = std::numeric_limits::max(); + bool enable_strict_mode = false; + IColumn::Filter* conversion_failure_null_map = nullptr; + IColumn::Filter dictionary_conversion_failures; + bool capturing_dictionary_conversion_failures = false; + bool dictionary_has_conversion_failures = false; + size_t dictionary_failure_scan_rows = 0; + ParquetDictionaryMaterializationStrategy dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; + + void reset_dictionary() { + typed_dictionary.reset(); + dictionary_indices.clear(); + dictionary_conversion_failures.clear(); + capturing_dictionary_conversion_failures = false; + dictionary_has_conversion_failures = false; + dictionary_failure_scan_rows = 0; + dictionary_generation = std::numeric_limits::max(); + } + + bool can_insert_null_on_conversion_failure() const { + return conversion_failure_null_map != nullptr && + (!enable_strict_mode || capturing_dictionary_conversion_failures); + } + + bool mark_conversion_failure(size_t output_row) { + if (!can_insert_null_on_conversion_failure()) { + return false; + } + DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); + (*conversion_failure_null_map)[output_row] = 1; + if (capturing_dictionary_conversion_failures) { + dictionary_has_conversion_failures = true; + } + return true; + } + + IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { + auto* output_null_map = conversion_failure_null_map; + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + dictionary_has_conversion_failures = false; + conversion_failure_null_map = &dictionary_conversion_failures; + capturing_dictionary_conversion_failures = true; + return output_null_map; + } + + void end_dictionary_conversion(IColumn::Filter* output_null_map) { + conversion_failure_null_map = output_null_map; + capturing_dictionary_conversion_failures = false; + } + + Status materialize_dictionary(IColumn& column) { + const size_t old_size = column.size(); + dictionary_failure_scan_rows = 0; + if (UNLIKELY(dictionary_has_conversion_failures)) { + const bool insert_failure_as_null = can_insert_null_on_conversion_failure(); + if (!insert_failure_as_null) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] == 0) { + continue; + } + // A malformed dictionary entry is irrelevant until a selected row references + // it; failing while building the dictionary would reject valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } + } + } + column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), + dictionary_indices.data() + dictionary_indices.size()); + if (UNLIKELY(dictionary_has_conversion_failures && + can_insert_null_on_conversion_failure())) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0) { + mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); + } + + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, size_t num_values) { + DORIS_CHECK(typed_dictionary); + if (UNLIKELY(dictionary_has_conversion_failures) || + source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::INDICES; + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); + DORIS_CHECK_EQ(dictionary_indices.size(), num_values); + return materialize_dictionary(column); + } + + class ColumnConsumer final : public ParquetDictionaryValueConsumer { + public: + ColumnConsumer(IColumn& destination, const IColumn& dictionary) + : _destination(destination), _dictionary(dictionary) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + _destination.insert_indices_from(_dictionary, indices, indices + num_values); + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + _destination.insert_many_from(_dictionary, index, num_values); + return Status::OK(); + } + + private: + IColumn& _destination; + const IColumn& _dictionary; + } consumer(column, *typed_dictionary); + + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::DIRECT; + const size_t old_size = column.size(); + const Status status = source.decode_dictionary_values(num_values, consumer); + if (!status.ok()) { + // Streaming direct gather may discover a corrupt late ID after earlier valid runs; + // restore the same all-or-nothing column invariant as the index-buffer path. + column.resize(old_size); + } + return status; + } +}; + +} // namespace doris diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h new file mode 100644 index 00000000000000..ba2aa686ad4272 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { + +#pragma pack(1) +struct ParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(ParquetInt96Timestamp) == 12); + +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; + +inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } + return Status::OK(); +} + +inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { + if (unit == ParquetTimeUnit::MILLIS) { + // Validate in the source unit before scaling; signed overflow could otherwise turn an + // invalid file value into an in-range timestamp that survives later range checks. + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + } else if (unit == ParquetTimeUnit::NANOS) { + *result = value / 1000; + // Epoch-relative negative timestamps must round toward the preceding representable + // microsecond; C++ division truncates toward zero and would move pre-epoch values forward. + if (value < 0 && value % 1000 != 0) { + --*result; + } + } else { + *result = value; + } + return validate_parquet_timestamp_micros(*result); +} + +inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000LL; + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + // INT96 nanos is a time-of-day, not a signed duration. Validate it before widened day + // arithmetic so corrupt input cannot wrap into a plausible Doris timestamp. + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + const __int128 timestamp_micros = + days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + } + *result = static_cast(timestamp_micros); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 35f29f2b54c471..d8fc1643d3756f 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -339,7 +339,7 @@ class FileScanner : public Scanner { return TPushAggOp::type::NONE; } return _effective_push_down_agg_type(_local_state->get_push_down_agg_type(), - _local_state->get_push_down_count_slot_ids()); + _local_state->get_push_down_count_slot_ids()); } // enable the file meta cache only when diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index b0c08bcce27552..e9d5b67dfa27f1 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -65,6 +65,7 @@ #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "service/backend_options.h" #include "storage/id_manager.h" @@ -319,26 +320,42 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _get_block_timer = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); - _empty_file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT, 1); - _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "NotFoundFileNum", TUnit::UNIT, 1); - _file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); - _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadBytes", TUnit::BYTES, 1); - _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadCalls", TUnit::UNIT, 1); + auto* profile = _local_state->scanner_profile(); + const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); + _scanner_total_timer = hierarchy.scanner; + _io_timer = hierarchy.io; + _init_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2InitTime", + file_scan_profile::SCANNER, 1); + _open_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2OpenTime", + file_scan_profile::SCANNER, 1); + _get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetBlockTime", + file_scan_profile::SCANNER, 1); + _prepare_split_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2PrepareSplitTime", + file_scan_profile::SCANNER, 1); + _get_next_range_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetNextRangeTime", + file_scan_profile::SCANNER, 1); + _close_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2CloseTime", + file_scan_profile::SCANNER, 1); + _empty_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptyFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _not_found_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NotFoundFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNumber", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_read_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadBytes", TUnit::BYTES, + file_scan_profile::IO, 1); + _file_read_calls_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadCalls", TUnit::UNIT, + file_scan_profile::IO, 1); _file_read_time_counter = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); - _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); - _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); - _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReadTime", file_scan_profile::IO, 1); + _adaptive_batch_predicted_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchPredictedRows", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _adaptive_batch_actual_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); + _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); _file_reader_stats = std::make_unique(); RETURN_IF_ERROR(_init_io_ctx()); @@ -349,6 +366,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc } Status FileScannerV2::_open_impl(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_open_timer); RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(Scanner::_open_impl(state)); RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); @@ -362,6 +381,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { } Status FileScannerV2::_get_next_scan_range(bool* has_next) { + SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); if (*has_next) { @@ -371,6 +391,8 @@ Status FileScannerV2::_get_next_scan_range(bool* has_next) { } Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); if (!_has_prepared_split) { @@ -381,7 +403,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - SCOPED_TIMER(_get_block_timer); if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -422,7 +443,23 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } } +Status FileScannerV2::_filter_output_block(Block* block) { + return _contextualize_output_filter_status(Scanner::_filter_output_block(block), + _get_current_format_type()); +} + +Status FileScannerV2::_contextualize_output_filter_status(Status status, + TFileFormatType::type format_type) { + if (!status.ok() && format_type == TFileFormatType::FORMAT_ORC) { + // Error-preserving expressions cannot be reordered into the ORC reader and therefore run + // at the scanner boundary; keep their error context identical to ORC callback failures. + status.prepend("Orc row reader nextBatch failed. reason = "); + } + return status; +} + Status FileScannerV2::_prepare_next_split(bool* eos) { + SCOPED_TIMER(_prepare_split_timer); while (true) { bool has_next = _first_scan_range; if (!_first_scan_range) { @@ -902,6 +939,8 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { } Status FileScannerV2::close(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_close_timer); if (_is_closed) { return Status::OK(); } @@ -1038,14 +1077,25 @@ void FileScannerV2::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); if (config::enable_file_cache && _state->query_options().enable_file_cache && _profile != nullptr) { - _report_file_cache_profile(_profile, *_file_cache_statistics); + auto file_cache_delta = io::diff_file_cache_statistics(*_file_cache_statistics, + _reported_file_cache_statistics); + // Profile collection can run more than once, so publish only the new additive delta. + _report_file_cache_profile(_profile, file_cache_delta); _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( - _file_cache_statistics->bytes_write_into_cache); + file_cache_delta.bytes_write_into_cache); + _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto read_time = cast_set(_file_reader_stats->read_time_ns); + DORIS_CHECK(read_time >= _reported_io_read_time); + // Some transports (for example Arrow Flight) record directly into IO, while filesystem + // reads arrive through FileReaderStats. Add only the new traced delta so both paths remain + // visible without double counting repeated profile publication. + COUNTER_UPDATE(_io_timer, read_time - _reported_io_read_time); + _reported_io_read_time = read_time; } // Query profiles can be collected before Scanner::close() runs. Publish condition-cache // counters here as well, using deltas so this method and close() cannot double count. @@ -1054,7 +1104,8 @@ void FileScannerV2::_collect_profile_before_close() { void FileScannerV2::_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { - io::FileCacheProfileReporter cache_profile(profile); + file_scan_profile::ensure_hierarchy(profile); + io::FileCacheProfileReporter cache_profile(profile, file_scan_profile::IO); cache_profile.update(&file_cache_statistics); } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 0cd03030b56851..fc217506eea0d3 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -88,6 +88,10 @@ class FileScannerV2 final : public Scanner { RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); + static Status TEST_contextualize_output_filter_status(Status status, + TFileFormatType::type format_type) { + return _contextualize_output_filter_status(std::move(status), format_type); + } static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, bool current_split_uses_metadata_count) { return _should_run_adaptive_batch_size(predictor_initialized, @@ -110,6 +114,7 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; + Status _filter_output_block(Block* block) override; void _collect_profile_before_close() override; bool _should_update_load_counters() const override; @@ -128,6 +133,8 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); static bool _should_skip_empty(const Status& status, bool stopped); + static Status _contextualize_output_filter_status(Status status, + TFileFormatType::type format_type); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -185,12 +192,20 @@ class FileScannerV2 final : public Scanner { std::unordered_map _partition_slot_descs; std::unique_ptr _file_cache_statistics; + io::FileCacheStatistics _reported_file_cache_statistics; std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + RuntimeProfile::Counter* _scanner_total_timer = nullptr; + RuntimeProfile::Counter* _init_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _prepare_split_timer = nullptr; + RuntimeProfile::Counter* _get_next_range_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _io_timer = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; @@ -207,6 +222,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_read_rows = 0; int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; + int64_t _reported_io_read_time = 0; }; } // namespace doris diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index 92119f7d3fff3a..e90754db1c23d6 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -137,7 +137,7 @@ class Scanner { virtual bool _should_update_load_counters() const { return _is_load; } // Filter the output block finally. - Status _filter_output_block(Block* block); + virtual Status _filter_output_block(Block* block); Status _do_projections(Block* origin_block, Block* output_block); diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp new file mode 100644 index 00000000000000..a0d4403086af53 --- /dev/null +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/runtime_filter_expr.h" + +#include + +#include + +#include "common/logging.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "exec/common/util.hpp" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_profile.h" +#include "storage/index/zone_map/zonemap_eval_context.h" + +namespace doris { + +class RowDescriptor; +class RuntimeState; +class TExprNode; +// branch-4.1 still builds VRuntimeFilterWrapper, which owns these shared threshold helpers. +} // namespace doris + +namespace doris { + +class VExprContext; + +RuntimeFilterExpr::RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, + bool null_aware, int filter_id, int sampling_frequency) + : VExpr(node), + _impl(std::move(impl)), + _ignore_thredhold(ignore_thredhold), + _null_aware(null_aware), + _filter_id(filter_id), + _sampling_frequency(sampling_frequency) { + DORIS_CHECK(_impl != nullptr); +} + +Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { + DORIS_CHECK(cloned_expr != nullptr); + DORIS_CHECK(_impl != nullptr); + VExprSPtr cloned_impl; + RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl)); + auto cloned_runtime_filter = RuntimeFilterExpr::create_shared( + clone_texpr_node(), std::move(cloned_impl), _ignore_thredhold, _null_aware, _filter_id, + _sampling_frequency); + cloned_runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, + _always_true_filter_rows); + *cloned_expr = std::move(cloned_runtime_filter); + return Status::OK(); +} + +Status RuntimeFilterExpr::prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) { + RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); + _expr_name = fmt::format("RuntimeFilterExpr({})", _impl->expr_name()); + _prepare_finished = true; + return Status::OK(); +} + +Status RuntimeFilterExpr::open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) { + DCHECK(_prepare_finished); + RETURN_IF_ERROR(_impl->open(state, context, scope)); + context->get_runtime_filter_selectivity().set_sampling_frequency(_sampling_frequency); + _open_finished = true; + return Status::OK(); +} + +void RuntimeFilterExpr::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { + _impl->close(context, scope); +} + +Status RuntimeFilterExpr::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + // branch-4.1's VExpr API predates const selectors; RuntimeFilterExpr only forwards the + // selector and does not mutate it. + return _impl->execute_column(context, block, const_cast(selector), count, + result_column); +} + +const std::string& RuntimeFilterExpr::expr_name() const { + return _expr_name; +} + +Status RuntimeFilterExpr::execute_filter(VExprContext* context, const Block* block, + uint8_t* __restrict result_filter_data, size_t rows, + bool accept_null, bool* can_filter_all) const { + DCHECK(_open_finished); + if (accept_null) { + return Status::InternalError( + "Runtime filter does not support accept_null in execute_filter"); + } + + auto& rf_selectivity = context->get_runtime_filter_selectivity(); + Defer auto_update_judge_counter = [&]() { rf_selectivity.update_judge_counter(); }; + + // if always true, skip evaluate runtime filter + if (rf_selectivity.maybe_always_true_can_ignore()) { + COUNTER_UPDATE(_always_true_filter_rows, rows); + return Status::OK(); + } + + ColumnPtr filter_column; + ColumnPtr arg_column = nullptr; + + RETURN_IF_ERROR(_impl->execute_runtime_filter(context, block, result_filter_data, rows, + filter_column, &arg_column)); + + // bloom filter will handle null aware inside itself + if (_null_aware && TExprNodeType::BLOOM_PRED != node_type()) { + DCHECK(arg_column); + change_null_to_true(filter_column->assert_mutable(), arg_column); + } + + if (const auto* const_column = check_and_get_column(*filter_column)) { + // const(nullable) or const(bool) + if (!const_column->get_bool(0)) { + // filter all + COUNTER_UPDATE(_rf_filter_rows, rows); + COUNTER_UPDATE(_rf_input_rows, rows); + rf_selectivity.update_judge_selectivity(_filter_id, rows, rows, _ignore_thredhold); + *can_filter_all = true; + memset(result_filter_data, 0, rows); + return Status::OK(); + } else { + // filter none + COUNTER_UPDATE(_rf_input_rows, rows); + rf_selectivity.update_judge_selectivity(_filter_id, 0, rows, _ignore_thredhold); + return Status::OK(); + } + } else if (const auto* nullable_column = check_and_get_column(*filter_column)) { + // nullable(bool) + const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr(); + const IColumn::Filter& filter = assert_cast(*nested_column).get_data(); + const auto* __restrict filter_data = filter.data(); + const auto* __restrict null_map_data = nullable_column->get_null_map_data().data(); + + const size_t input_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + for (size_t i = 0; i < rows; ++i) { + result_filter_data[i] &= (!null_map_data[i]) & filter_data[i]; + } + + const size_t output_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + COUNTER_UPDATE(_rf_filter_rows, input_rows - output_rows); + COUNTER_UPDATE(_rf_input_rows, input_rows); + rf_selectivity.update_judge_selectivity(_filter_id, input_rows - output_rows, input_rows, + _ignore_thredhold); + + if (output_rows == 0) { + *can_filter_all = true; + return Status::OK(); + } + } else { + // bool + const IColumn::Filter& filter = assert_cast(*filter_column).get_data(); + const auto* __restrict filter_data = filter.data(); + + const size_t input_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + for (size_t i = 0; i < rows; ++i) { + result_filter_data[i] &= filter_data[i]; + } + + const size_t output_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + COUNTER_UPDATE(_rf_filter_rows, input_rows - output_rows); + COUNTER_UPDATE(_rf_input_rows, input_rows); + rf_selectivity.update_judge_selectivity(_filter_id, input_rows - output_rows, input_rows, + _ignore_thredhold); + + if (output_rows == 0) { + *can_filter_all = true; + return Status::OK(); + } + } + return Status::OK(); +} + +ZoneMapFilterResult RuntimeFilterExpr::evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx) const { + if (_null_aware) { + const auto child = _impl->get_child(0); + if (auto slot_ref = std::dynamic_pointer_cast(child); slot_ref != nullptr) { + auto zone_map = ctx.zone_map(slot_ref->column_id()); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + + if (zone_map->has_null) { + return ZoneMapFilterResult::kMayMatch; + } + } else { + // RuntimeFilterExpr implementations generated by RuntimeFilterConsumer put the probe + // slot as child 0. If a custom RF expression violates this shape, keep the null-aware + // evaluation conservative because we cannot decide whether the target zone has NULLs. + return unsupported_zonemap_filter(ctx); + } + } + return _impl->evaluate_zonemap_filter(ctx); +} + +bool RuntimeFilterExpr::can_evaluate_zonemap_filter() const { + return _impl->can_evaluate_zonemap_filter(); +} + +void RuntimeFilterExpr::collect_slot_column_ids(std::set& column_ids) const { + _impl->collect_slot_column_ids(column_ids); +} + +} // namespace doris diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h new file mode 100644 index 00000000000000..7994d2a71ae14f --- /dev/null +++ b/be/src/exprs/runtime_filter_expr.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "exec/runtime_filter/runtime_filter_selectivity.h" +#include "exprs/function_context.h" +#include "exprs/vexpr.h" +#include "runtime/runtime_profile.h" + +namespace doris { +class RowDescriptor; +class RuntimeState; +class TExprNode; + +double get_in_list_ignore_thredhold(size_t list_size); +double get_comparison_ignore_thredhold(); +double get_bloom_filter_ignore_thredhold(); +} // namespace doris + +namespace doris { + +class Block; +class VExprContext; + +class RuntimeFilterExpr final : public VExpr { + ENABLE_FACTORY_CREATOR(RuntimeFilterExpr); + +public: + RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, + bool null_aware, int filter_id, + int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING); + ~RuntimeFilterExpr() override = default; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override; + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override; + std::string debug_string() const override { return _impl->debug_string(); } + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + const std::string& expr_name() const override; + const VExprSPtrs& children() const override { return _impl->children(); } + TExprNodeType::type node_type() const override { return _impl->node_type(); } + + double execute_cost() const override { return _impl->execute_cost(); } + + Status execute_filter(VExprContext* context, const Block* block, + uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, + bool* can_filter_all) const override; + + uint64_t get_digest(uint64_t seed) const override { + seed = _impl->get_digest(seed); + if (seed) { + return HashUtil::crc_hash64(&_null_aware, sizeof(_null_aware), seed); + } + return seed; + } + + VExprSPtr get_impl() const override { return _impl; } + void set_impl(VExprSPtr impl) { _impl = std::move(impl); } + Status clone_node(VExprSPtr* cloned_expr) const override; + + void attach_profile_counter(std::shared_ptr rf_input_rows, + std::shared_ptr rf_filter_rows, + std::shared_ptr always_true_filter_rows) { + DCHECK(rf_input_rows != nullptr); + DCHECK(rf_filter_rows != nullptr); + DCHECK(always_true_filter_rows != nullptr); + + if (rf_input_rows != nullptr) { + _rf_input_rows = rf_input_rows; + } + if (rf_filter_rows != nullptr) { + _rf_filter_rows = rf_filter_rows; + } + if (always_true_filter_rows != nullptr) { + _always_true_filter_rows = always_true_filter_rows; + } + } + + bool is_rf_wrapper() const override { return true; } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; + bool can_evaluate_zonemap_filter() const override; + void collect_slot_column_ids(std::set& column_ids) const override; + + int filter_id() const { return _filter_id; } + + std::shared_ptr predicate_filtered_rows_counter() const { + return _rf_filter_rows; + } + std::shared_ptr predicate_input_rows_counter() const { + return _rf_input_rows; + } + std::shared_ptr predicate_always_true_rows_counter() const { + return _always_true_filter_rows; + } + bool is_slot_ref() const override { return false; } + bool is_virtual_slot_ref() const override { return false; } + bool is_column_ref() const override { return false; } + +private: + VExprSPtr _impl; + + std::shared_ptr _rf_input_rows = + std::make_shared(TUnit::UNIT, 0); + std::shared_ptr _rf_filter_rows = + std::make_shared(TUnit::UNIT, 0); + std::shared_ptr _always_true_filter_rows = + std::make_shared(TUnit::UNIT, 0); + + std::string _expr_name; + double _ignore_thredhold; + bool _null_aware; + int _filter_id; + int _sampling_frequency; +}; + +using RuntimeFilterExprPtr = std::shared_ptr; + +} // namespace doris diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index 8bf0839f274d1a..770d7a8d5f59a0 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -128,6 +128,14 @@ class VCompoundPred : public VectorizedFnCall { } } + bool is_safe_to_execute_on_selected_rows() const override { + // Boolean composition introduces no data-dependent failure of its own. Reuse the generic + // child walk so AND/OR remain eligible only when every nested expression is independently + // safe; applying VectorizedFnCall's scalar-function allowlist to this structural node would + // incorrectly disable selected-row execution for otherwise safe predicates. + return VExpr::is_safe_to_execute_on_selected_rows(); + } + ZoneMapFilterResult evaluate_dictionary_filter( const DictionaryEvalContext& ctx) const override { switch (_op) { diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 621d183fc84a05..58e3af4ad4c87b 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include +#include "common/compare.h" #include "common/config.h" #include "common/exception.h" #include "common/logging.h" @@ -83,6 +85,82 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; +namespace { + +enum class RawComparisonOp : uint8_t { EQ, NE, LT, LE, GT, GE }; + +std::optional raw_comparison_op(std::string_view function_name, bool reverse) { + RawComparisonOp op; + if (function_name == "eq") { + op = RawComparisonOp::EQ; + } else if (function_name == "ne") { + op = RawComparisonOp::NE; + } else if (function_name == "lt") { + op = RawComparisonOp::LT; + } else if (function_name == "le") { + op = RawComparisonOp::LE; + } else if (function_name == "gt") { + op = RawComparisonOp::GT; + } else if (function_name == "ge") { + op = RawComparisonOp::GE; + } else { + return std::nullopt; + } + if (!reverse || op == RawComparisonOp::EQ || op == RawComparisonOp::NE) { + return op; + } + switch (op) { + case RawComparisonOp::LT: + return RawComparisonOp::GT; + case RawComparisonOp::LE: + return RawComparisonOp::GE; + case RawComparisonOp::GT: + return RawComparisonOp::LT; + case RawComparisonOp::GE: + return RawComparisonOp::LE; + case RawComparisonOp::EQ: + case RawComparisonOp::NE: + break; + } + __builtin_unreachable(); +} + +template +void execute_raw_comparison(const uint8_t* values, size_t num_values, const Field& literal, + RawComparisonOp op, uint8_t* matches) { + const T rhs = literal.get(); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T lhs = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (op) { + case RawComparisonOp::EQ: + keep = Compare::equal(lhs, rhs); + break; + case RawComparisonOp::NE: + keep = Compare::not_equal(lhs, rhs); + break; + case RawComparisonOp::LT: + keep = Compare::less(lhs, rhs); + break; + case RawComparisonOp::LE: + keep = Compare::less_equal(lhs, rhs); + break; + case RawComparisonOp::GT: + keep = Compare::greater(lhs, rhs); + break; + case RawComparisonOp::GE: + keep = Compare::greater_equal(lhs, rhs); + break; + } + matches[row] = static_cast(keep); + } +} + +} // namespace + VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { _function_name = _fn.name.function_name; } @@ -339,6 +417,71 @@ Status VectorizedFnCall::execute_column(VExprContext* context, const Block* bloc return _do_execute(context, block, selector, count, result_column, nullptr); } +bool VectorizedFnCall::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + if (data_type == nullptr || !raw_comparison_op(_function_name, false).has_value()) { + return false; + } + auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + if (!slot_literal.has_value() || slot_literal->slot_index != column_id || + slot_literal->literal.is_null()) { + return false; + } + const auto raw_type = remove_nullable(data_type); + if (!remove_nullable(slot_literal->slot_type)->equals(*raw_type) || + !remove_nullable(slot_literal->literal_type)->equals(*raw_type)) { + return false; + } + const auto primitive_type = raw_type->get_primitive_type(); + return primitive_type == TYPE_INT || primitive_type == TYPE_BIGINT || + primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE; +} + +Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const { + if (!can_execute_on_raw_fixed_values(data_type, column_id)) { + return Status::NotSupported("Expression {} cannot evaluate raw fixed-width values", + expr_name()); + } + DORIS_CHECK(values != nullptr || num_values == 0); + DORIS_CHECK(matches != nullptr || num_values == 0); + const auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + DORIS_CHECK(slot_literal.has_value()); + const auto op = raw_comparison_op(_function_name, slot_literal->literal_on_left); + DORIS_CHECK(op.has_value()); + const auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + const size_t expected_width = primitive_type == TYPE_INT || primitive_type == TYPE_FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + if (value_width != expected_width) { + return Status::Corruption("Raw expression width {} does not match expected {}", value_width, + expected_width); + } + switch (primitive_type) { + case TYPE_INT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_BIGINT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_FLOAT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_DOUBLE: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + default: + __builtin_unreachable(); + } + return Status::OK(); +} + const std::string& VectorizedFnCall::expr_name() const { return _expr_name; } @@ -386,8 +529,12 @@ bool VectorizedFnCall::is_deterministic() const { } bool VectorizedFnCall::is_safe_to_execute_on_selected_rows() const { - static const std::set ERROR_PRESERVING_FUNCTIONS = {"assert_true"}; - return !ERROR_PRESERVING_FUNCTIONS.contains(_function_name) && + static const std::set TOTAL_PREDICATE_FUNCTIONS = { + "eq", "ne", "lt", "le", "gt", "ge", "in", "not_in", "is_null_pred", "is_not_null_pred"}; + // Selected-row execution may hide data-dependent errors in rows rejected by an earlier + // predicate. Keep function calls unsafe by default and opt in only operations that are total + // for their input domain; child checks then reject expressions such as gt(mod(x, -1), 0). + return TOTAL_PREDICATE_FUNCTIONS.contains(_function_name) && VExpr::is_safe_to_execute_on_selected_rows(); } diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 765113f0b1cafc..749133a098c625 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -58,6 +58,11 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_filter(VExprContext* context, const Block* block, const uint8_t* __restrict filter, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const override; + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override; + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const override; Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index e7f9851b61a44e..dd61d299a7fe0e 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -164,6 +165,15 @@ class VExpr { return execute_column_impl(context, block, selector, count, result_column); } + template , int> = 0> + Status execute_column(VExprContext* context, const Block* block, SelectorType* selector, + size_t count, ColumnPtr& result_column) const { + // format_v2 treats selectors as read-only; the branch-4.1 expression API still exposes + // a mutable pointer even though implementations must not modify it. + return execute_column(context, block, const_cast(selector), count, + result_column); + } + virtual Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const { @@ -179,6 +189,19 @@ class VExpr { uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, bool* can_filter_all) const; + // Raw fixed-width evaluation is an optional expression capability used before a storage reader + // materializes a column. `matches` is ANDed in place; callers handle NULL rows separately + // because raw value streams contain only non-NULL payloads. + virtual bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + return false; + } + virtual Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, const DataTypePtr& data_type, + int column_id, uint8_t* matches) const { + return Status::NotSupported("{} cannot evaluate raw fixed-width values", expr_name()); + } + // `is_blockable` means this expr will be blocked in `execute` (e.g. AI Function, Remote Function) [[nodiscard]] virtual bool is_blockable() const { return std::any_of(_children.begin(), _children.end(), diff --git a/be/src/format/file_reader/new_plain_text_line_reader.cpp b/be/src/format/file_reader/new_plain_text_line_reader.cpp index a4df306bf49db5..25a7a0a7ac2476 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/format/file_reader/new_plain_text_line_reader.cpp @@ -45,7 +45,6 @@ // leave these 2 size small for debugging namespace doris { -#include "common/compile_check_begin.h" const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, const size_t length) { if (_skip_utf8_bom && !_first_record_prefix_checked && _idx == 0) { constexpr uint8_t UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; @@ -54,8 +53,12 @@ const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, con if (std::memcmp(start, UTF8_BOM, prefix_size) != 0) { _first_record_prefix_checked = true; } else if (length < UTF8_BOM_SIZE) { + // The input buffer can end inside the BOM. Wait for the remaining prefix bytes instead + // of feeding a partial BOM into START, which would permanently select NORMAL state. return nullptr; } else { + // Keep offsets relative to the original buffer. CsvReader removes these three bytes + // from the returned line and shifts separator positions by the same amount. _idx = UTF8_BOM_SIZE; _first_record_prefix_checked = true; } @@ -64,7 +67,10 @@ const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, con // causing parse column separator error. if (_state.curr_state == ReaderState::NORMAL || _state.curr_state == ReaderState::MATCH_ENCLOSE) { - _idx -= std::min(_column_sep_len - 1, _idx); + const size_t last_column_sep_end = + _column_sep_positions.empty() ? 0 : _column_sep_positions.back() + _column_sep_len; + DORIS_CHECK_LE(last_column_sep_end, _idx); + _idx -= std::min(_column_sep_len - 1, _idx - last_column_sep_end); } _total_len = length; size_t bound = update_reading_bound(start); @@ -148,7 +154,8 @@ void EncloseCsvLineReaderCtx::_on_start(const uint8_t* start, size_t& len) { void EncloseCsvLineReaderCtx::_on_normal(const uint8_t* start, size_t& len) { const uint8_t* curr_start = start + _idx; - size_t curr_len = len - _idx; + const size_t field_search_bound = _result == nullptr ? len : _result - start; + const size_t curr_len = field_search_bound - _idx; const uint8_t* col_sep_pos = find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len); @@ -204,7 +211,8 @@ void EncloseCsvLineReaderCtx::_on_pre_match_enclose(const uint8_t* start, size_t void EncloseCsvLineReaderCtx::_on_match_enclose(const uint8_t* start, size_t& len) { const uint8_t* curr_start = start + _idx; - size_t curr_len = len - _idx; + const size_t field_search_bound = _result == nullptr ? len : _result - start; + const size_t curr_len = field_search_bound - _idx; const uint8_t* delim_pos = find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len); @@ -541,5 +549,4 @@ void NewPlainTextLineReader::_collect_profile_before_close() { } } -#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/format/file_reader/new_plain_text_line_reader.h b/be/src/format/file_reader/new_plain_text_line_reader.h index 3f18f28c673ad1..ed7f80493b02d4 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.h +++ b/be/src/format/file_reader/new_plain_text_line_reader.h @@ -29,7 +29,6 @@ #include "runtime/runtime_profile.h" namespace doris { -#include "common/compile_check_begin.h" namespace io { struct IOContext; } @@ -213,9 +212,11 @@ class EncloseCsvLineReaderCtx final : public BaseTextLineReaderContext +#include + +#include "roaring/roaring64map.hh" + +namespace doris { + +// Doris materializes an Iceberg deletion vector as one buffer. This is an implementation limit, +// not an Iceberg format limit. +inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30; +inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12; + +// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum serialized size is below this +// limit. Keep its guard independent from the Iceberg capability limit. +inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30; + +// A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the +// query-local cache instead of expanding every set bit into an int64_t. Position delete files use +// a different representation because their input is a stream of (file_path, row_position) rows. +using DeletionVector = roaring::Roaring64Map; + +} // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index a193c552447a34..7ee77e94eff255 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -40,6 +40,7 @@ #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" #include "format/table/deletion_vector_reader.h" +#include "format/table/iceberg_reader.h" #include "format/table/table_format_reader.h" #include "io/hdfs_builder.h" #include "runtime/runtime_state.h" @@ -135,8 +136,8 @@ Status visit_position_delete_block(const Block& block, size_t read_rows, return Status::InternalError("Unsupported file_path column type in position delete block"); } -Status init_parquet_delete_reader(ParquetReader* reader) { - if (reader == nullptr) { +Status init_parquet_delete_reader(ParquetReader* reader, bool* dictionary_coded) { + if (reader == nullptr || dictionary_coded == nullptr) { return Status::InvalidArgument("invalid parquet delete reader arguments"); } @@ -146,6 +147,20 @@ Status init_parquet_delete_reader(ParquetReader* reader) { slot_id_to_predicates, nullptr, nullptr, nullptr, nullptr, nullptr, TableSchemaChangeHelper::ConstNode::get_instance(), false)); + // branch-4.1 initializes its projected-column state separately from init_reader(). Skipping + // this step makes every row group look like a path-only scan and yields empty batches forever. + RETURN_IF_ERROR(reader->set_fill_columns({}, {})); + + const tparquet::FileMetaData* meta_data = reader->get_meta_data(); + *dictionary_coded = true; + for (const auto& row_group : meta_data->row_groups) { + const auto& column_chunk = row_group.columns[0]; + if (!(column_chunk.__isset.meta_data && + IcebergTableReader::_is_fully_dictionary_encoded(column_chunk.meta_data))) { + *dictionary_coded = false; + break; + } + } return Status::OK(); } @@ -282,19 +297,30 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi options.batch_size, const_cast(&options.state->timezone_obj()), options.io_ctx, options.state, options.meta_cache); - RETURN_IF_ERROR(init_parquet_delete_reader(&reader)); + bool dictionary_coded = false; + RETURN_IF_ERROR(init_parquet_delete_reader(&reader, &dictionary_coded)); bool eof = false; while (!eof) { Block block; - // branch-4.1 cannot safely nest ColumnDictI32 in ColumnNullable. Decode paths to - // strings so optional delete columns keep their null map for corruption checks. - block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), - ICEBERG_FILE_PATH)); + if (dictionary_coded) { + block.insert(ColumnWithTypeAndName( + ColumnNullable::create(ColumnDictI32::create(), ColumnUInt8::create()), + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); + } else { + block.insert(ColumnWithTypeAndName( + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); + } block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), ICEBERG_ROW_POS)); size_t read_rows = 0; RETURN_IF_ERROR(reader.get_next_block(&block, &read_rows, &eof)); + if (read_rows == 0 && !eof) { + // Position-delete scans have no predicates, so an empty non-EOF batch cannot + // make progress and must not spin forever on a malformed reader state. + return Status::InternalError( + "Parquet position delete reader returned an empty non-EOF batch"); + } RETURN_IF_ERROR(visit_position_delete_block(block, read_rows, visitor)); } return Status::OK(); diff --git a/be/src/format/table/paimon_jni_reader.cpp b/be/src/format/table/paimon_jni_reader.cpp index 063755d2be58b7..772c97e2aa6067 100644 --- a/be/src/format/table/paimon_jni_reader.cpp +++ b/be/src/format/table/paimon_jni_reader.cpp @@ -85,17 +85,14 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d params[PAIMON_OPTION_PREFIX + kv.first] = kv.second; } } - const std::string enable_io_manager_key = - PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; - const std::string io_manager_tmp_dir_key = - PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; + const std::string enable_io_manager_key = PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; + const std::string io_manager_tmp_dir_key = PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; auto enable_io_manager_it = params.find(enable_io_manager_key); if (enable_io_manager_it != params.end() && iequal(enable_io_manager_it->second, "true") && params.find(io_manager_tmp_dir_key) == params.end()) { std::vector tmp_dirs; for (const auto& store_path : state->exec_env()->store_paths()) { - tmp_dirs.push_back(store_path.path + "/" + - std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + tmp_dirs.push_back(store_path.path + "/" + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); } DORIS_CHECK(!tmp_dirs.empty()); // Paimon owns its paimon-* children; Doris only supplies stable storage-root parents. diff --git a/be/src/format/table/paimon_reader.cpp b/be/src/format/table/paimon_reader.cpp index df0741b3e11920..9c292cdbde224b 100644 --- a/be/src/format/table/paimon_reader.cpp +++ b/be/src/format/table/paimon_reader.cpp @@ -28,8 +28,7 @@ namespace doris { #include "common/compile_check_begin.h" -std::string build_paimon_deletion_vector_cache_key( - const TPaimonDeletionFileDesc& deletion_file) { +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file) { // A shared file can contain multiple vectors, so the cache identity must include its range. return deletion_file.path + "#" + std::to_string(deletion_file.offset) + "#" + std::to_string(deletion_file.length); @@ -46,6 +45,47 @@ Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& bytes_read); } +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector) { + if (deletion_vector == nullptr) { + return Status::InvalidArgument("deletion vector output must not be null"); + } + if (buf == nullptr) { + return Status::DataQualityError("Paimon deletion vector blob is null"); + } + if (buffer_size < 8) [[unlikely]] { + return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); + } + + const uint32_t actual_length = BigEndian::Load32(buf); + if (static_cast(actual_length) + 4 != buffer_size) [[unlikely]] { + return Status::DataQualityError( + "Paimon deletion vector length mismatch, expected: {}, actual: {}", + static_cast(actual_length) + 4, buffer_size); + } + + // Paimon deletion vectors always prefix the portable Roaring payload with this magic value. + constexpr char paimon_bitmap_magic[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + if (memcmp(buf + sizeof(actual_length), paimon_bitmap_magic, 4) != 0) [[unlikely]] { + return Status::DataQualityError( + "Paimon deletion vector magic number mismatch, expected: {}, actual: {}", + BigEndian::Load32(paimon_bitmap_magic), + BigEndian::Load32(buf + sizeof(actual_length))); + } + + roaring::Roaring roaring_bitmap; + try { + roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8); + } catch (const std::runtime_error& e) { + return Status::RuntimeError( + "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", + e.what()); + } + + *deletion_vector |= DeletionVector(std::move(roaring_bitmap)); + return Status::OK(); +} + PaimonReader::PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, const TFileRangeDesc& range, diff --git a/be/src/format/table/paimon_reader.h b/be/src/format/table/paimon_reader.h index 566841d947d093..e08d8bb59a8629 100644 --- a/be/src/format/table/paimon_reader.h +++ b/be/src/format/table/paimon_reader.h @@ -23,17 +23,20 @@ #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" +#include "format/table/deletion_vector.h" #include "format/table/table_format_reader.h" namespace doris { #include "common/compile_check_begin.h" -std::string build_paimon_deletion_vector_cache_key( - const TPaimonDeletionFileDesc& deletion_file); +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, size_t& bytes_read); +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector); + class PaimonReader : public TableFormatReader, public TableSchemaChangeHelper { public: PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 377a26ce530db8..e417617cf5a64f 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -108,6 +108,107 @@ instructions as well; this file adds format-v2-specific review expectations. - For JNI readers, review local/global reference lifetime, exception propagation, type conversion, thread attachment assumptions, and cleanup on partial initialization. +### Parquet Native Decode Kernel + +- Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline, but v2 owns an independent page/encoding reader and must not call the v1 + `ParquetColumnReader`. Do not modify `be/src/format/parquet/` for a v2 decoder change. Reimplement + the required behavior under the v2 tree and keep v1 unchanged so differential correctness and + performance results remain meaningful. +- Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema + objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, + and Dremel-level thresholds. Review constructor arguments and stored references for ownership and + lifetime; metadata owned by a temporary schema adapter must not escape into a persistent reader. +- Treat selection positions as logical Row Group rows, including null rows. Selection indices must + be sorted, unique, and bounded by the batch's logical row count. Dense identity, empty selection, + and fragmented selection must have explicit representations and tests; do not silently mix row + ordinals with non-null value ordinals or dictionary IDs. +- Verify the three decode counts separately: logical rows consumed, encoded non-null payload values + consumed, and output values materialized. Null and filtered-null runs consume no payload; + selected and filtered non-null runs both consume payload. Every page transition, skip, error, and + end-of-batch path must leave all three cursors aligned for the next call. +- A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a + definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Choose one physical leaf as the parent + shape owner; sibling leaf streams advance over the same parent-row range and validate their + payload counts instead of independently redefining the parent shape. +- The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the + nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded + batch or a stateful load-before-build phase. Ordinary predicate/output scans construct + `NativeColumnReader`, consume compressed page data through the native decoder, and append directly + into the final Doris column. +- Keep logical schema changes distinct from physical decoding. The native reader materializes the + projected file type only; `ColumnMapper` and `TableReader` own every file-to-table cast. A type + mismatch at the native reader boundary is an invariant violation, not a reason to create a + reader-local conversion path or expose a decoder-owned value batch. +- `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key + for MAP) and advances only definition/repetition levels. It exposes no value API and must not be + reused as a scan reader or expanded into a fallback path. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one traversal + of the owning leaf's levels. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning that leaf. MAP key levels own entry + existence; value levels validate against that shape. STRUCT siblings validate parent-row + alignment. If every projected STRUCT child is missing, consume only a retained physical leaf's + levels and never materialize its payload into a temporary Doris column. +- Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row + can contain more level entries than the requested parent-row batch. Split large runs without + changing alternation or row-boundary semantics, and check overflow before narrowing counts. +- Decoder dispatch must reject an incompatible physical type, encoding, or type length explicitly. + Review PLAIN, dictionary/RLE, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, BOOLEAN RLE, and level RLE/bit-packed paths for identical + selection and malformed-input behavior. Page V1 and V2 must feed the same decoder contract after + their different level/decompression layouts are parsed. +- Keep every index coordinate domain explicit: table-local column ID, physical leaf-column ID, + Row Group ID, data-page ordinal, OffsetIndex row ordinal, logical batch-row ordinal, non-null + payload ordinal, and dictionary-entry ID are different types of identity. Data-page ordinals must + exclude dictionary pages consistently. Dictionary-entry bitmaps are local to one Column Chunk + dictionary and cannot be reused after a Row Group, dictionary, or encoding transition. +- Review index composition, not only each index in isolation. Row Group statistics, dictionary, + Bloom, ColumnIndex/OffsetIndex, page cache registration, page skip plans, SelectionVector, and + lazy column cursors must describe the same surviving logical rows. Missing or unusable optional + indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an + explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID + filtering before any cursor is consumed. +- Decoder owns encoded-stream parsing and cursor movement only. `DataTypeSerDe` owns Parquet + physical/logical interpretation and writes directly into Doris columns. Dictionary pages are + materialized once through the same SerDe and data pages expose only validated dictionary indices. + Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, + precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, + unsupported physical types, encodings, page layouts, or malformed inputs return an explicit + status. Arrow may be used by metadata planning and as a test oracle; no Arrow array, builder, + RecordReader, or metadata lifetime belongs in data-page value or level materialization. +- Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across + batches. String-like decoders should gather selected `StringRef` values and append once per batch, + rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded + high-water mark but must be reset logically between pages, Row Groups, files, and errors. +- Adaptive batch sizing must measure completed Doris output rows/bytes and must not recreate native + readers, builders, or scratch when only the row cap changes. Compare the v1 and v2 lifecycle: + probe batches must not turn persistent setup into a per-batch cost or amplify highly fragmented + selection work. +- Footer and metadata caching must key on stable file identity and cache the serialized footer plus + parsed native metadata at the same lifecycle as v1. Never reuse metadata when path, file size, + modification/version identity, encryption state, or schema-affecting options differ. Cache misses + and uncacheable identities must remain correct without a fallback to stale entries. +- Page-cache behavior must match v1 for cache key, stable file identity, registered byte range, + compressed/decompressed entry kind, checksum/decompression ownership, subrange coverage, + invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory + evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, + decoder, level, or dictionary cursor state. +- Apply v1's MergeRange decision to the native data-page reader, after metadata/dictionary probes + finish. Predicate and lazy readers for one Row Group must share one ordered-range wrapper, and + native per-column prefetch must be disabled while that wrapper is active. Never allocate one + MergeRange buffer per leaf; wide complex projections would multiply its bounded scratch memory. +- Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, + dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, + selection, conversion, allocation, and materialization time must remain attributable without + adding per-row timer overhead. +- Profile index attempts, successes, conservative fallbacks, and corrupt rejections separately for + statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skipping. Footer/page/file/ + condition-cache counters must expose requests, hits, misses, writes/admissions, bytes, wait/I/O + time, and bypass reasons with semantics aligned to v1. A lower total timer without its internal + work counters is not sufficient observability. + ## Detailed FileReader Review Guides - Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, @@ -174,6 +275,15 @@ instructions as well; this file adds format-v2-specific review expectations. malformed input. - For bug fixes, require a test that fails for the original reachable path and validates the result, row count, or explicit error after the fix. +- For native Parquet work, require a matrix across physical/logical types, every supported encoding, + Page V1/V2, required/optional/repeated levels, dictionary fallback, page and batch boundaries, + dense/empty/fragmented selections, null placement, malformed/truncated input, and files from + representative external writers. Include focused cursor-invariant tests and end-to-end nested + reconstruction tests; a scalar happy-path benchmark is not sufficient coverage. +- Performance-sensitive decoder changes need a reproducible comparison against v1 and a relevant + external baseline such as DuckDB. Report data shape, encoding, selectivity, null/cardinality + distribution, compression, storage path, batch policy, warm/cold cache state, CPU time, rows/s, + bytes/s, allocation behavior, and Profile counter deltas. ## Review Output diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index e2d1fba8b3a8b4..7457098a76feec 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -37,6 +37,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/short_circuit_evaluation_expr.h" #include "exprs/vcase_expr.h" #include "exprs/vcast_expr.h" @@ -45,7 +46,6 @@ #include "exprs/vexpr_context.h" #include "exprs/vin_predicate.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "format_v2/column_mapper_nested.h" #include "format_v2/expr/cast.h" #include "format_v2/file_reader.h" @@ -372,14 +372,23 @@ static bool is_cast_expr(const VExprSPtr& expr) { } static bool is_binary_comparison_predicate(const VExprSPtr& expr) { - // BINARY_PRED and NULL_AWARE_BINARY_PRED are comparison-only node kinds. Nereids does not - // always populate the legacy opcode after resolving the comparison through TFunction, so - // requiring expr->op() to be set leaves an otherwise ordinary slot-literal predicate with - // mismatched implicit-coercion types. Row evaluation still works through the resolved - // function, but Parquet zonemap evaluation conservatively rejects the mismatched types. - return expr != nullptr && expr->get_num_children() == 2 && - (expr->node_type() == TExprNodeType::BINARY_PRED || - expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED); + if (expr == nullptr || expr->get_num_children() != 2 || + (expr->node_type() != TExprNodeType::BINARY_PRED && + expr->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED)) { + return false; + } + switch (expr->op()) { + case TExprOpcode::EQ: + case TExprOpcode::EQ_FOR_NULL: + case TExprOpcode::NE: + case TExprOpcode::GE: + case TExprOpcode::GT: + case TExprOpcode::LE: + case TExprOpcode::LT: + return true; + default: + return false; + } } std::string TableColumnMapperOptions::debug_string() const { @@ -514,8 +523,20 @@ static bool table_filter_has_only_local_entries( return true; } -static bool is_lossless_file_to_table_numeric_cast(const DataTypePtr& file_type, - const DataTypePtr& table_type); +static VExprSPtr unwrap_literal_for_file_cast(const VExprSPtr& expr, + const DataTypePtr& table_type) { + if (expr == nullptr) { + return nullptr; + } + if (expr->is_literal()) { + return expr; + } + if (is_cast_expr(expr) && expr->get_num_children() == 1 && expr->children()[0]->is_literal() && + expr->children()[0]->data_type()->equals(*table_type)) { + return expr->children()[0]; + } + return nullptr; +} static Field literal_field_from_expr(const VExpr& literal_expr) { DORIS_CHECK(literal_expr.is_literal()); @@ -526,48 +547,6 @@ static Field literal_field_from_expr(const VExpr& literal_expr) { return field; } -static VExprSPtr unwrap_literal_for_file_cast(const VExprSPtr& expr, const DataTypePtr& table_type, - RewriteContext* rewrite_context) { - if (expr == nullptr) { - return nullptr; - } - - VExprSPtr literal; - if (expr->is_literal()) { - literal = expr; - } else if (is_cast_expr(expr) && expr->get_num_children() == 1 && - expr->children()[0]->is_literal() && expr->data_type()->equals(*table_type)) { - literal = expr->children()[0]; - } else { - return nullptr; - } - - if (literal->data_type()->equals(*table_type)) { - return literal; - } - // Nereids may leave a narrow numeric literal directly under a comparison and rely on the - // comparison's implicit type coercion, for example `INT slot = TINYINT 1`. Materialize that - // coercion before localizing the predicate so metadata readers still see slot-literal form. - if (!is_lossless_file_to_table_numeric_cast(literal->data_type(), table_type)) { - return nullptr; - } - - Field table_field; - try { - convert_field_to_type(literal_field_from_expr(*literal), *table_type, &table_field, - literal->data_type().get()); - } catch (const Exception&) { - return nullptr; - } - if (table_field.is_null() || - table_field.get_type() != remove_nullable(table_type)->get_primitive_type()) { - return nullptr; - } - auto normalized_literal = VLiteral::create_shared(table_type, std::move(table_field)); - rewrite_context->add_created_expr(normalized_literal); - return normalized_literal; -} - // Table filter localization clones an already-prepared table expr and then rewrites it to file // slots. Only split-local literals and BE cast nodes need table-reader-specific clone behavior; // plain slot refs and literals use their own VExpr::clone_node(). @@ -771,8 +750,8 @@ static bool rewrite_binary_slot_literal_predicate( if (rewrite_info == nullptr || slot_ref == nullptr) { return false; } - auto literal_expr = unwrap_literal_for_file_cast(children[literal_child_idx], - rewrite_info->table_type, rewrite_context); + auto literal_expr = + unwrap_literal_for_file_cast(children[literal_child_idx], rewrite_info->table_type); if (literal_expr == nullptr) { return false; } @@ -809,8 +788,8 @@ static bool rewrite_in_slot_literal_predicate( VExprSPtrs rewritten_literals; rewritten_literals.reserve(children.size() - 1); for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { - auto literal_expr = unwrap_literal_for_file_cast(children[child_idx], - rewrite_info->table_type, rewrite_context); + auto literal_expr = + unwrap_literal_for_file_cast(children[child_idx], rewrite_info->table_type); if (literal_expr == nullptr) { return false; } @@ -818,8 +797,8 @@ static bool rewrite_in_slot_literal_predicate( rewrite_literal_to_file_type(literal_expr, *rewrite_info, rewrite_context); if (rewritten_literal == nullptr) { for (size_t restore_idx = 1; restore_idx < children.size(); ++restore_idx) { - auto restore_literal = unwrap_literal_for_file_cast( - children[restore_idx], rewrite_info->table_type, rewrite_context); + auto restore_literal = unwrap_literal_for_file_cast(children[restore_idx], + rewrite_info->table_type); if (restore_literal != nullptr) { children[restore_idx] = original_table_literal(restore_literal, rewrite_context); @@ -1014,8 +993,7 @@ static bool rewrite_binary_struct_literal_predicate( const auto table_leaf_type = children[struct_child_idx]->data_type(); DORIS_CHECK(table_leaf_type != nullptr); - auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type, - rewrite_context); + auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type); if (table_literal == nullptr || !rewrite_struct_element_path_to_file_expr(children[struct_child_idx], filter_mappings, global_to_file_slot, rewrite_context)) { @@ -1069,8 +1047,7 @@ static bool rewrite_in_struct_literal_predicate( VExprSPtrs table_literals; table_literals.reserve(children.size() - 1); for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { - auto table_literal = - unwrap_literal_for_file_cast(children[child_idx], table_leaf_type, rewrite_context); + auto table_literal = unwrap_literal_for_file_cast(children[child_idx], table_leaf_type); if (table_literal == nullptr) { return false; } @@ -1185,7 +1162,7 @@ static VExprSPtr rewrite_table_expr_to_file_expr( } DORIS_CHECK(rewrite_context != nullptr); DORIS_CHECK(can_localize != nullptr); - if (auto* runtime_filter = dynamic_cast(expr.get()); + if (auto* runtime_filter = dynamic_cast(expr.get()); runtime_filter != nullptr) { auto impl = runtime_filter->get_impl(); if (impl == nullptr) { @@ -2184,6 +2161,7 @@ Status TableColumnMapper::create_scan_request( // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); + file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); @@ -2215,6 +2193,30 @@ Status TableColumnMapper::create_scan_request( // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + for (const auto& mapping : _hidden_mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + const bool is_visible_output = + std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { + return visible_mapping.file_local_id.has_value() && + LocalColumnId(*visible_mapping.file_local_id) == local_id; + }); + if (is_visible_output) { + continue; + } + // File-local filtering is an optimization; Scanner still evaluates the original + // table-level conjunct after TableReader returns. Only truly hidden mappings are absent + // from that scanner-visible block and may safely discard their payload here. + if (std::ranges::any_of(file_request->predicate_columns, + [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }) && + !file_request->is_predicate_only(local_id)) { + file_request->predicate_only_columns.push_back(local_id); + } + } // 3. Rebuild output projection expressions for projected columns. localize_filters() has // already applied the final scan projection to mapping.file_type/projected_file_children before // rewriting filter expressions. @@ -2294,8 +2296,16 @@ Status TableColumnMapper::localize_filters(const std::vector& table filter_mappings = _filter_visible_mappings(); const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); for (const auto& table_filter : table_filters) { - if (table_filter.conjunct != nullptr && - table_filter_has_only_local_entries(table_filter, _filter_entries)) { + if (table_filter.conjunct != nullptr && table_filter.conjunct->root() != nullptr) { + const auto root = table_filter.conjunct->root(); + const auto impl = root->get_impl(); + const auto predicate = impl != nullptr ? impl : root; + if (!predicate->is_deterministic() || + !table_filter_has_only_local_entries(table_filter, _filter_entries)) { + continue; + } + // Scanner evaluates the original conjunct after final materialization. Only predicates + // whose result is stable across repeated execution may also run as a file-local copy. RewriteContext rewrite_context {.runtime_state = runtime_state}; VExprSPtr rewrite_root; Status clone_status; diff --git a/be/src/format_v2/deletion_vector.h b/be/src/format_v2/deletion_vector.h deleted file mode 100644 index 12ec7dbc6ebc80..00000000000000 --- a/be/src/format_v2/deletion_vector.h +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "roaring/roaring64map.hh" - -namespace doris { - -// Doris materializes an Iceberg deletion vector as one buffer. This is an implementation limit, -// not an Iceberg format limit. -inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30; -inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12; - -// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum serialized size is below this -// limit. Keep its guard independent from the Iceberg capability limit. -inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30; - -// A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the -// query-local cache instead of expanding every set bit into an int64_t. Position delete files use -// a different representation because their input is a stream of (file_path, row_position) rows. -using DeletionVector = roaring::Roaring64Map; - -} // namespace doris diff --git a/be/src/format_v2/deletion_vector_reader.cpp b/be/src/format_v2/deletion_vector_reader.cpp deleted file mode 100644 index d70956de8b60fc..00000000000000 --- a/be/src/format_v2/deletion_vector_reader.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/deletion_vector_reader.h" - -#include - -#include -#include - -#include "exec/common/endian.h" - -namespace doris::format { - -std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, - const TIcebergDeleteFileDesc& delete_file) { - return fmt::format("delete_dv_{}:{}{}:{}#{}#{}", data_file_path.size(), data_file_path, - delete_file.path.size(), delete_file.path, delete_file.content_offset, - delete_file.content_size_in_bytes); -} - -Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* rows_to_delete) { - if (buf == nullptr || rows_to_delete == nullptr) { - return Status::InvalidArgument("invalid deletion vector decode arguments"); - } - if (buffer_size < 12) { - return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); - } - - const auto total_length = BigEndian::Load32(buf); - if (total_length + 8 != buffer_size) { - return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", - total_length + 8, buffer_size); - } - - constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; - if (std::memcmp(buf + sizeof(total_length), MAGIC_NUMBER, 4) != 0) { - return Status::DataQualityError("Deletion vector magic number mismatch"); - } - - try { - *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); - } catch (const std::runtime_error& e) { - return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); - } - return Status::OK(); -} - -std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file) { - return fmt::format("paimon_dv_{}#{}#{}", deletion_file.path, deletion_file.offset, - deletion_file.length); -} - -Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* deletion_vector) { - if (deletion_vector == nullptr) { - return Status::InvalidArgument("deletion_vector must not be null"); - } - if (buffer_size < 8) [[unlikely]] { - return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); - } - const uint32_t actual_length = BigEndian::Load32(buf); - if (actual_length + 4 != buffer_size) [[unlikely]] { - return Status::RuntimeError( - "DeletionVector deserialize error: length not match, actual length: {}, expect " - "length: {}", - actual_length, buffer_size - 4); - } - constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; - if (std::memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { - return Status::RuntimeError("DeletionVector deserialize error: invalid magic number {}", - BigEndian::Load32(buf + sizeof(actual_length))); - } - roaring::Roaring roaring_bitmap; - try { - roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8); - } catch (const std::runtime_error& e) { - return Status::RuntimeError( - "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", - e.what()); - } - *deletion_vector |= DeletionVector(std::move(roaring_bitmap)); - return Status::OK(); -} - -DeletionVectorReader::~DeletionVectorReader() { - _file_reader.reset(); - _merge_io_statistics(); -} - -void DeletionVectorReader::_init_io_context() { - if (_parent_io_ctx == nullptr) { - return; - } - _reader_io_ctx = *_parent_io_ctx; - _reader_io_ctx.file_cache_stats = &_file_cache_stats; - _reader_io_ctx.file_reader_stats = &_file_reader_stats; - _io_ctx = &_reader_io_ctx; -} - -void DeletionVectorReader::_merge_io_statistics() { - if (_statistics_merged || _parent_io_ctx == nullptr) { - return; - } - if (_parent_io_ctx->file_cache_stats != nullptr) { - _parent_io_ctx->file_cache_stats->merge_from(_file_cache_stats); - } - if (_parent_io_ctx->file_reader_stats != nullptr) { - _parent_io_ctx->file_reader_stats->read_calls += _file_reader_stats.read_calls; - _parent_io_ctx->file_reader_stats->read_bytes += _file_reader_stats.read_bytes; - _parent_io_ctx->file_reader_stats->read_time_ns += _file_reader_stats.read_time_ns; - _parent_io_ctx->file_reader_stats->read_rows += _file_reader_stats.read_rows; - } - _statistics_merged = true; -} - -Status DeletionVectorReader::open() { - if (_is_opened) [[unlikely]] { - return Status::OK(); - } - - _init_system_properties(); - _init_file_description(); - RETURN_IF_ERROR(_create_file_reader()); - - _file_size = _file_reader->size(); - _is_opened = true; - return Status::OK(); -} - -Status DeletionVectorReader::read_at(size_t offset, Slice result) { - if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { - return Status::EndOfFile("stop read."); - } - if (_io_ctx != nullptr) { - _io_ctx->should_stop = _parent_io_ctx->should_stop; - } - size_t bytes_read = 0; - RETURN_IF_ERROR(_file_reader->read_at(offset, result, &bytes_read, _io_ctx)); - if (bytes_read != result.size) [[unlikely]] { - return Status::IOError("Failed to read fully at offset {}, expected {}, got {}", offset, - result.size, bytes_read); - } - return Status::OK(); -} - -Status DeletionVectorReader::_create_file_reader() { - if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { - return Status::EndOfFile("stop read."); - } - - _file_description.mtime = _desc.modification_time; - io::FileReaderOptions reader_options = - FileFactory::get_reader_options(_state->query_options(), _file_description); - _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( - _profile, _system_properties, _file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx)); - return Status::OK(); -} - -void DeletionVectorReader::_init_file_description() { - _file_description.path = _desc.path; - _file_description.file_size = _desc.file_size; - _file_description.fs_name = _desc.fs_name; -} - -void DeletionVectorReader::_init_system_properties() { - _system_properties.system_type = _params.file_type; - _system_properties.properties = _params.properties; - _system_properties.hdfs_params = _params.hdfs_params; - if (_params.__isset.broker_addresses) { - _system_properties.broker_addresses.assign(_params.broker_addresses.begin(), - _params.broker_addresses.end()); - } -} - -} // namespace doris::format diff --git a/be/src/format_v2/deletion_vector_reader.h b/be/src/format_v2/deletion_vector_reader.h deleted file mode 100644 index 7ecdb6f1ab219f..00000000000000 --- a/be/src/format_v2/deletion_vector_reader.h +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "common/status.h" -#include "format/generic_reader.h" -#include "format_v2/deletion_vector.h" -#include "gen_cpp/PlanNodes_types.h" -#include "io/file_factory.h" -#include "io/fs/buffered_reader.h" -#include "io/fs/file_reader.h" -#include "io/io_common.h" -#include "util/profile_collector.h" -#include "util/slice.h" - -namespace doris::format { - -struct DeleteFileDesc { - enum class Format { - PAIMON, - ICEBERG, - }; - - std::string key = ""; - std::string path = ""; - std::string fs_name = ""; - int64_t start_offset = 0; - int64_t size = 0; - int64_t file_size = -1; - int64_t modification_time = 0; - Format format = Format::PAIMON; -}; - -std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, - const TIcebergDeleteFileDesc& delete_file); - -Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* rows_to_delete); - -std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); - -Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* deletion_vector); - -class DeletionVectorReader { -public: - DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, - const TFileScanRangeParams& params, const DeleteFileDesc& desc, - io::IOContext* io_ctx) - : _state(state), - _profile(profile), - _params(params), - _desc(desc), - _parent_io_ctx(io_ctx) { - _init_io_context(); - } - ~DeletionVectorReader(); - - DeletionVectorReader(const DeletionVectorReader&) = delete; - DeletionVectorReader& operator=(const DeletionVectorReader&) = delete; - - Status open(); - Status read_at(size_t offset, Slice result); - - const io::FileCacheStatistics& file_cache_statistics() const { return _file_cache_stats; } - -private: - void _init_system_properties(); - void _init_file_description(); - void _init_io_context(); - void _merge_io_statistics(); - Status _create_file_reader(); - - RuntimeState* _state = nullptr; - RuntimeProfile* _profile = nullptr; - const TFileScanRangeParams& _params; - DeleteFileDesc _desc; - io::IOContext* _parent_io_ctx = nullptr; - io::IOContext _reader_io_ctx; - io::IOContext* _io_ctx = nullptr; - io::FileCacheStatistics _file_cache_stats; - io::FileReaderStats _file_reader_stats; - bool _statistics_merged = false; - - io::FileSystemProperties _system_properties; - io::FileDescription _file_description; - io::FileReaderSPtr _file_reader; - int64_t _file_size = 0; - bool _is_opened = false; -}; - -} // namespace doris::format diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp b/be/src/format_v2/delimited_text/csv_reader.cpp index 5935e9323f71b6..bb6c55d30847b2 100644 --- a/be/src/format_v2/delimited_text/csv_reader.cpp +++ b/be/src/format_v2/delimited_text/csv_reader.cpp @@ -17,7 +17,6 @@ #include "format_v2/delimited_text/csv_reader.h" -#include #include #include @@ -34,226 +33,6 @@ #include "util/utf8_check.h" namespace doris::format::csv { - -enum class CsvReaderState { START, NORMAL, PRE_MATCH_ENCLOSE, MATCH_ENCLOSE }; - -struct CsvReaderStateWrapper { - void forward_to(CsvReaderState state) { - prev_state = curr_state; - curr_state = state; - } - - void reset() { - curr_state = CsvReaderState::START; - prev_state = CsvReaderState::START; - } - - CsvReaderState curr_state = CsvReaderState::START; - CsvReaderState prev_state = CsvReaderState::START; -}; - -// The v1 CSV context cannot distinguish a column separator that overlaps the line delimiter and -// may rewind into a separator that was already emitted when the output buffer is refilled. Keep -// the stricter behavior local to format v2 so the v1 reader remains unchanged. -class EncloseCsvLineReaderV2Ctx final - : public BaseTextLineReaderContext { - using FindDelimiterFunc = const uint8_t* (*)(const uint8_t*, size_t, const char*, size_t); - -public: - EncloseCsvLineReaderV2Ctx(const std::string& line_delimiter_, size_t line_delimiter_len_, - std::string column_separator, size_t column_separator_len, - size_t column_separator_count, char enclose, char escape, - bool keep_cr_, bool skip_utf8_bom) - : BaseTextLineReaderContext(line_delimiter_, line_delimiter_len_, keep_cr_), - _enclose(enclose), - _escape(escape), - _skip_utf8_bom(skip_utf8_bom), - _column_separator_len(column_separator_len), - _column_separator(std::move(column_separator)) { - if (_column_separator_len == 1) { - _find_column_separator = &look_for_column_separator; - } else { - _find_column_separator = &look_for_column_separator; - } - _column_separator_positions.reserve(column_separator_count); - } - - void refresh_impl() { - _idx = 0; - _should_escape = false; - _quote_escape = false; - _result = nullptr; - _column_separator_positions.clear(); - _state.reset(); - } - - [[nodiscard]] const std::vector& column_sep_positions() const { - return _column_separator_positions; - } - - void adjust_column_sep_positions(size_t offset) { - for (auto& position : _column_separator_positions) { - position -= offset; - } - } - - const uint8_t* read_line_impl(const uint8_t* start, size_t length) { - if (_skip_utf8_bom && !_first_record_prefix_checked && _idx == 0) { - constexpr uint8_t UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; - constexpr size_t UTF8_BOM_SIZE = sizeof(UTF8_BOM); - const size_t prefix_size = std::min(length, UTF8_BOM_SIZE); - if (std::memcmp(start, UTF8_BOM, prefix_size) != 0) { - _first_record_prefix_checked = true; - } else if (length < UTF8_BOM_SIZE) { - return nullptr; - } else { - _idx = UTF8_BOM_SIZE; - _first_record_prefix_checked = true; - } - } - - if (_state.curr_state == CsvReaderState::NORMAL || - _state.curr_state == CsvReaderState::MATCH_ENCLOSE) { - const size_t last_separator_end = - _column_separator_positions.empty() - ? 0 - : _column_separator_positions.back() + _column_separator_len; - DORIS_CHECK_LE(last_separator_end, _idx); - _idx -= std::min(_column_separator_len - 1, _idx - last_separator_end); - } - - _total_len = length; - size_t bound = update_reading_bound(start); - while (_idx != bound) { - switch (_state.curr_state) { - case CsvReaderState::START: - on_start(start); - break; - case CsvReaderState::NORMAL: - on_normal(start, bound); - break; - case CsvReaderState::PRE_MATCH_ENCLOSE: - on_pre_match_enclose(start, bound); - break; - case CsvReaderState::MATCH_ENCLOSE: - on_match_enclose(start, bound); - break; - } - } - return _result; - } - -private: - template - static const uint8_t* look_for_column_separator(const uint8_t* start, size_t length, - const char* separator, size_t separator_len) { - if constexpr (SingleChar) { - for (size_t i = 0; i < length; ++i) { - if (start[i] == separator[0]) { - return start + i; - } - } - return nullptr; - } - return static_cast(memmem(start, length, separator, separator_len)); - } - - size_t update_reading_bound(const uint8_t* start) { - _result = call_find_line_sep(start + _idx, _total_len - _idx); - return _result == nullptr ? _total_len : _result - start + line_delimiter_length(); - } - - void on_column_separator_found(const uint8_t* start, const uint8_t* separator_position) { - const uint8_t* field_start = start + _idx; - _column_separator_positions.push_back(separator_position - start); - _idx += separator_position + _column_separator_len - field_start; - } - - void on_start(const uint8_t* start) { - if (start[_idx] == _enclose) [[unlikely]] { - _state.forward_to(CsvReaderState::PRE_MATCH_ENCLOSE); - ++_idx; - } else { - _state.forward_to(CsvReaderState::NORMAL); - } - } - - void on_normal(const uint8_t* start, size_t bound) { - const size_t search_bound = _result == nullptr ? bound : _result - start; - DORIS_CHECK_LE(_idx, search_bound); - const uint8_t* separator_position = - _find_column_separator(start + _idx, search_bound - _idx, _column_separator.c_str(), - _column_separator_len); - if (separator_position != nullptr) [[likely]] { - on_column_separator_found(start, separator_position); - _state.forward_to(CsvReaderState::START); - return; - } - _idx = bound; - } - - void on_pre_match_enclose(const uint8_t* start, size_t& bound) { - do { - do { - if (_escape != _enclose && start[_idx] == _escape) [[unlikely]] { - _should_escape = !_should_escape; - } else if (_should_escape) [[unlikely]] { - _should_escape = false; - } else if (_quote_escape) { - if (start[_idx] == _enclose) { - _quote_escape = false; - } else { - _quote_escape = false; - _state.forward_to(CsvReaderState::MATCH_ENCLOSE); - return; - } - } else if (start[_idx] == _enclose) { - _quote_escape = true; - } else { - _quote_escape = false; - } - ++_idx; - } while (_idx != bound); - - if (_idx != _total_len) { - bound = update_reading_bound(start); - } else { - _result = nullptr; - break; - } - } while (true); - } - - void on_match_enclose(const uint8_t* start, size_t bound) { - const size_t search_bound = _result == nullptr ? bound : _result - start; - DORIS_CHECK_LE(_idx, search_bound); - const uint8_t* separator_position = - _find_column_separator(start + _idx, search_bound - _idx, _column_separator.c_str(), - _column_separator_len); - if (separator_position != nullptr) [[likely]] { - on_column_separator_found(start, separator_position); - _state.forward_to(CsvReaderState::START); - return; - } - _idx = bound; - } - - CsvReaderStateWrapper _state; - const char _enclose; - const char _escape; - const bool _skip_utf8_bom; - bool _first_record_prefix_checked = false; - const uint8_t* _result = nullptr; - size_t _total_len = 0; - const size_t _column_separator_len; - size_t _idx = 0; - bool _should_escape = false; - bool _quote_escape = false; - const std::string _column_separator; - std::vector _column_separator_positions; - FindDelimiterFunc _find_column_separator; -}; - namespace { bool starts_with_at(const Slice& line, size_t pos, const std::string& needle) { @@ -357,7 +136,7 @@ Status CsvReader::_create_line_reader() { } else { const size_t col_sep_num = _source_file_slot_descs.size() > 1 ? _source_file_slot_descs.size() - 1 : 0; - _enclose_reader_ctx = std::make_shared( + _enclose_reader_ctx = std::make_shared( _line_delimiter, _line_delimiter.size(), _value_separator, _value_separator.size(), col_sep_num, _enclose, _escape, _keep_cr, _start_offset == 0); diff --git a/be/src/format_v2/delimited_text/csv_reader.h b/be/src/format_v2/delimited_text/csv_reader.h index 088da8ba77cd50..dd1d77d5a34b9a 100644 --- a/be/src/format_v2/delimited_text/csv_reader.h +++ b/be/src/format_v2/delimited_text/csv_reader.h @@ -25,13 +25,12 @@ #include "util/slice.h" namespace doris { +class EncloseCsvLineReaderCtx; class SlotDescriptor; } // namespace doris namespace doris::format::csv { -class EncloseCsvLineReaderV2Ctx; - // FileScannerV2 CSV reader. // // CSV files do not carry a physical schema. FE provides the table slot descriptors plus @@ -71,7 +70,7 @@ class CsvReader final : public ::doris::format::DelimitedTextReader { bool _trim_tailing_spaces = false; bool _empty_field_as_null = false; bool _keep_cr = false; - std::shared_ptr _enclose_reader_ctx; + std::shared_ptr _enclose_reader_ctx; }; } // namespace doris::format::csv diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index f19d12c75714b9..63486d174effe0 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -38,6 +38,7 @@ #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/string_util.h" @@ -180,7 +181,9 @@ void DelimitedTextReader::_init_profile() { return; } - ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + file_scan_profile::ensure_hierarchy(_profile); + _text_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, + file_scan_profile::FILE_READER, 1); _text_profile.open_file_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); _text_profile.create_line_reader_time = @@ -199,8 +202,10 @@ void DelimitedTextReader::_init_profile() { _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + // RuntimeProfile counter names are global within one profile, so the format prefix preserves + // independent ownership when Parquet and delimited readers are initialized in either order. _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _profile, "DelimitedRowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -215,6 +220,7 @@ void DelimitedTextReader::_init_profile() { Status DelimitedTextReader::init(RuntimeState* state) { _init_profile(); + SCOPED_TIMER(_text_profile.total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); @@ -275,6 +281,7 @@ Status DelimitedTextReader::init(RuntimeState* state) { } Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_text_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); } @@ -288,6 +295,7 @@ std::unique_ptr DelimitedTextReader::create_column_mapper( } Status DelimitedTextReader::open(std::shared_ptr request) { + SCOPED_TIMER(_text_profile.total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -307,6 +315,7 @@ Status DelimitedTextReader::open(std::shared_ptr request) { } Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -358,11 +367,18 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, FileAggregateResult* result) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(result != nullptr); if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", _reader_name); } + if (!request.columns.empty()) { + // Text files expose no NULL-count metadata, and this fast path intentionally skips field + // parsing. Returning the physical row count for COUNT(nullable_col) would be incorrect. + return Status::NotSupported("{} v2 reader cannot push down COUNT with a column argument", + _reader_name); + } if (_line_reader == nullptr) { return Status::InternalError("{} v2 reader is not open", _reader_name); } @@ -396,6 +412,7 @@ Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& req } Status DelimitedTextReader::close() { + SCOPED_TIMER(_text_profile.total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h index daea3bc90942d8..dff27980c9cd12 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.h +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -59,6 +59,7 @@ class DelimitedTextReader : public FileReader { protected: struct DelimitedTextProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* open_file_time = nullptr; RuntimeProfile::Counter* create_line_reader_time = nullptr; RuntimeProfile::Counter* read_line_time = nullptr; diff --git a/be/src/format_v2/delimited_text/text_reader.cpp b/be/src/format_v2/delimited_text/text_reader.cpp index 02bd9cd92f8383..6dea7a07a4c6fa 100644 --- a/be/src/format_v2/delimited_text/text_reader.cpp +++ b/be/src/format_v2/delimited_text/text_reader.cpp @@ -25,7 +25,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type_serde/data_type_string_serde.h" #include "format/file_reader/new_plain_text_line_reader.h" -#include "format_v2/delimited_text/hive_text_util.h" +#include "format/hive_text_util.h" #include "runtime/descriptors.h" #include "util/decompressor.h" diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index 5dd8de557c062b..efeb9d851deb22 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -111,8 +111,7 @@ Status Cast::_do_execute(VExprContext* context, const Block* block, const Select ColumnNumbers args(1); ColumnPtr tmp_arg_column; - RETURN_IF_ERROR(_children[0]->execute_column(context, block, const_cast(selector), - count, tmp_arg_column)); + RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, tmp_arg_column)); auto arg_type = _children[0]->execute_type(block); temp_block.insert({tmp_arg_column, arg_type, _children[0]->expr_name()}); args[0] = 0; @@ -125,6 +124,7 @@ Status Cast::_do_execute(VExprContext* context, const Block* block, const Select num_columns_without_result, count)); result_column = temp_block.get_by_position(num_columns_without_result).column; DCHECK_EQ(result_column->size(), count); + RETURN_IF_ERROR(result_column->column_self_check()); return Status::OK(); } diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 42e50e7ba07364..1d111aa7a74930 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -26,28 +26,63 @@ #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, size_t rhs_row) { - if (lhs->is_nullable() && rhs->is_nullable()) { - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; - } + const IColumn* lhs_data = lhs.get(); + const IColumn* rhs_data = rhs.get(); if (lhs->is_nullable()) { const auto& nullable_lhs = assert_cast(*lhs); - return !nullable_lhs.is_null_at(lhs_row) && - nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + if (nullable_lhs.is_null_at(lhs_row)) { + return rhs->is_nullable() && + assert_cast(*rhs).is_null_at(rhs_row); + } + lhs_data = &nullable_lhs.get_nested_column(); } if (rhs->is_nullable()) { const auto& nullable_rhs = assert_cast(*rhs); - return !nullable_rhs.is_null_at(rhs_row) && - lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + if (nullable_rhs.is_null_at(rhs_row)) { + return false; + } + rhs_data = &nullable_rhs.get_nested_column(); + } + const bool lhs_binary = check_and_get_column(*lhs_data) != nullptr || + check_and_get_column(*lhs_data) != nullptr; + const bool rhs_binary = check_and_get_column(*rhs_data) != nullptr || + check_and_get_column(*rhs_data) != nullptr; + if (lhs_binary && rhs_binary) { + // Iceberg schema evolution may represent the same byte key as STRING in one file and + // VARBINARY in another. Equality-delete semantics compare bytes, not column storage classes. + return lhs_data->get_data_at(lhs_row) == rhs_data->get_data_at(rhs_row); + } + return lhs_data->compare_at(lhs_row, rhs_row, *rhs_data, -1) == 0; +} + +void update_varbinary_hashes(const ColumnWithTypeAndName& entry, uint64_t* hashes) { + const IColumn* data = entry.column.get(); + const uint8_t* null_map = nullptr; + if (entry.column->is_nullable()) { + const auto& nullable = assert_cast(*entry.column); + data = &nullable.get_nested_column(); + null_map = nullable.get_null_map_data().data(); + } + for (size_t row = 0; row < entry.column->size(); ++row) { + if (null_map != nullptr && null_map[row] != 0) { + hashes[row] = HashUtil::xxHash64NullWithSeed(hashes[row]); + continue; + } + const auto bytes = data->get_data_at(row); + hashes[row] = HashUtil::xxHash64WithSeed(bytes.data, bytes.size, hashes[row]); } - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; } } // namespace @@ -123,7 +158,7 @@ Status EqualityDeletePredicate::execute_column_impl(VExprContext* context, const Block data_key_block; for (const auto& child : _children) { ColumnPtr key_column; - RETURN_IF_ERROR(child->execute_column_impl(context, block, selector, count, key_column)); + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, key_column)); // Equality comparison operates on row-addressable columns. Materialize literal constants // so nullable NULL keys and regular slot columns share the same compare_at contract. data_key_block.insert({key_column->convert_to_full_column_if_const(), @@ -155,8 +190,14 @@ ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_blo std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { std::vector hashes(block.rows(), 0); - for (const auto& column : block.get_columns()) { - column->update_hashes_with_value(hashes.data(), nullptr); + for (const auto& entry : block) { + if (remove_nullable(entry.type)->get_primitive_type() == TYPE_VARBINARY) { + // ColumnVarbinary intentionally lacks the generic column hash hook. Keep the V2 delete + // hash byte-identical to ColumnString so schema-mapped binary keys share hash buckets. + update_varbinary_hashes(entry, hashes.data()); + } else { + entry.column->update_hashes_with_value(hashes.data(), nullptr); + } } return hashes; } diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index b2603a17958eae..9bbf7c3a8fc066 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -53,6 +53,10 @@ std::string FileScanRequest::debug_string() const { << join_debug_strings( non_predicate_columns, [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", predicate_only_columns=" + << join_debug_strings( + predicate_only_columns, + [](LocalColumnId column_id) { return std::to_string(column_id.value()); }) << ", local_positions={"; size_t position_idx = 0; for (const auto& [column_id, block_position] : local_positions) { diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 96d940702486b4..5f959c3e672dcd 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -70,6 +70,10 @@ struct FileScanRequest { // Columns read after row-level filtering. Predicate columns are also available for output and // should not be duplicated here. std::vector non_predicate_columns; + // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead + // after all file-local predicates run, although the shared file block still needs row-shaped + // placeholders until TableReader finalizes projected columns. + std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; // Row-level filters converted to file-local expressions from table-level predicates. @@ -89,6 +93,10 @@ struct FileScanRequest { return std::ranges::find(count_star_placeholder_columns, column_id) != count_star_placeholder_columns.end(); } + + bool is_predicate_only(LocalColumnId column_id) const { + return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/jni/iceberg_sys_table_reader.cpp b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp index 896929925aab3f..b41d505f886d31 100644 --- a/be/src/format_v2/jni/iceberg_sys_table_reader.cpp +++ b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp @@ -17,10 +17,33 @@ #include "format_v2/jni/iceberg_sys_table_reader.h" +#include + +#include "format/jni/jni_data_bridge.h" +#include "util/string_util.h" + namespace doris::format::iceberg { +namespace { + +constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop."; + +} // namespace Status IcebergSysTableJniReader::validate_scan_range(const TFileRangeDesc& range) const { - return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); + if (!range.__isset.table_format_params) { + return Status::InternalError( + "missing table_format_params for iceberg sys table jni reader"); + } + if (!range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("missing iceberg_params for iceberg sys table jni reader"); + } + if (!range.table_format_params.iceberg_params.__isset.serialized_split || + range.table_format_params.iceberg_params.serialized_split.empty()) { + return Status::InternalError( + "missing serialized_split for iceberg sys table jni reader, " + "possibly caused by FE/BE protocol mismatch"); + } + return Status::OK(); } std::string IcebergSysTableJniReader::connector_class() const { @@ -31,7 +54,23 @@ Status IcebergSysTableJniReader::build_scanner_params( std::map* params) const { DORIS_CHECK(params != nullptr); params->clear(); - return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); + params->emplace("serialized_split", + _current_range.table_format_params.iceberg_params.serialized_split); + + std::vector required_types; + required_types.reserve(_projected_columns.size()); + for (const auto& column : _projected_columns) { + required_types.emplace_back(JniDataBridge::get_jni_type_with_different_string(column.type)); + } + (*params)["required_types"] = join(required_types, "#"); + + if (_scan_params != nullptr && _scan_params->__isset.properties && + !_scan_params->properties.empty()) { + for (const auto& kv : _scan_params->properties) { + (*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second; + } + } + return Status::OK(); } } // namespace doris::format::iceberg diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index e9d225aeb46fc5..7d28134db4d7e3 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -30,6 +30,7 @@ #include "exprs/function/simple_function_factory.h" #include "exprs/vexpr_context.h" #include "format_v2/table_reader.h" +#include "util/jdbc_utils.h" namespace doris::format::jdbc { @@ -38,14 +39,34 @@ std::string JdbcJniReader::connector_class() const { } Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { - return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); + { + // End these scopes before JniTableReader enters the same counters; nested use would count + // this JDBC parameter preparation twice instead of extending the common lifecycle total. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + _jdbc_params.clear(); + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.table_format_type == "jdbc") { + _jdbc_params = std::map( + options.current_range.table_format_params.jdbc_params.begin(), + options.current_range.table_format_params.jdbc_params.end()); + } + } + return format::JniTableReader::prepare_split(options); } // need pass to the java side, so the java scanner can parse the params and construct the JDBC connection Status JdbcJniReader::build_scanner_params(std::map* params) const { DORIS_CHECK(params != nullptr); - params->clear(); - return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); + *params = _jdbc_params; + if (params->contains("jdbc_driver_url")) { + std::string resolved; + if (JdbcUtils::resolve_driver_url((*params)["jdbc_driver_url"], &resolved).ok()) { + (*params)["jdbc_driver_url"] = resolved; + } + } + return Status::OK(); } Status JdbcJniReader::build_jni_columns( diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 3cb105193b56ae..7658a52fc86e0f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "exprs/vexpr_context.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/string_util.h" @@ -31,17 +32,31 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); - _init_profile(); + { + // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers add + // deltas, so nested use would double-count instead of extending lifecycle coverage. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _init_profile(); + } + SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { - // EOF belongs to the previous split. Keep it set after closing that split so repeated reads - // are idempotent, and clear it only when a new split is explicitly prepared. - _eof = false; - _current_range = options.current_range; - RETURN_IF_ERROR(validate_scan_range(options.current_range)); + SCOPED_TIMER(_connector_total_time); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -63,6 +78,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { } Status JniTableReader::get_block(Block* output_block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + SCOPED_TIMER(_connector_total_time); DORIS_CHECK(output_block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(output_block->columns() == _projected_columns.size()); @@ -109,7 +127,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) { } Status JniTableReader::abort_split() { - RETURN_IF_ERROR(_close_jni_scanner()); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + RETURN_IF_ERROR(_close_jni_scanner()); + } return TableReader::abort_split(); } @@ -342,10 +364,16 @@ void JniTableReader::_publish_split_profile(JNIEnv* env) { } Status JniTableReader::close() { + SCOPED_TIMER(_connector_total_time); if (_closed) { return Status::OK(); } - auto close_status = _close_jni_scanner(); + Status close_status; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + close_status = _close_jni_scanner(); + } auto table_status = TableReader::close(); if (close_status.ok() && !table_status.ok()) { close_status = std::move(table_status); @@ -535,7 +563,9 @@ void JniTableReader::_init_profile() { return; } const auto connector_name = _connector_name(); - ADD_TIMER(_scanner_profile, connector_name); + file_scan_profile::ensure_hierarchy(_scanner_profile); + _connector_total_time = + ADD_CHILD_TIMER(_scanner_profile, connector_name, file_scan_profile::TABLE_READER); _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); _java_append_data_time = diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index fe908d654863eb..5e48270515f996 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -23,7 +23,7 @@ #include "common/status.h" #include "core/data_type/data_type.h" -#include "format_v2/jni/jni_data_bridge.h" +#include "format/jni/jni_data_bridge.h" #include "format_v2/table_reader.h" #include "runtime/runtime_profile.h" #include "util/jni-util.h" @@ -84,6 +84,7 @@ class JniTableReader : public TableReader { virtual Status _open_jni_scanner(); bool _reserve_split_profile_publication(); const std::vector& jni_columns() const { return _jni_columns; } + RuntimeProfile::Counter* connector_total_timer() const { return _connector_total_time; } TFileRangeDesc _current_range; private: @@ -110,6 +111,7 @@ class JniTableReader : public TableReader { bool _eof = false; bool _split_profile_published = false; + RuntimeProfile::Counter* _connector_total_time = nullptr; RuntimeProfile::Counter* _open_scanner_time = nullptr; RuntimeProfile::Counter* _java_scan_time = nullptr; RuntimeProfile::Counter* _java_append_data_time = nullptr; diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp index 11c9945c5dea16..4f0a3f55c3891e 100644 --- a/be/src/format_v2/jni/trino_connector_jni_reader.cpp +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -84,8 +84,15 @@ Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) } Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(validate_scan_range(options.current_range)); - RETURN_IF_ERROR(_set_spi_plugins_dir()); + { + // Plugin discovery can dominate a cold split. Use non-overlapping common scopes because + // the JNI base method subsequently enters the same RuntimeProfile counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 313115e82ce744..1f42a2cd2d5e94 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -47,6 +47,7 @@ #include "io/fs/stream_load_pipe.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/slice.h" @@ -174,7 +175,26 @@ JsonReader::~JsonReader() { static_cast(close()); } +void JsonReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* json_profile = "JsonReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, json_profile, file_scan_profile::FILE_READER, 1); + _open_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonOpenTime", json_profile, 1); + _read_document_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonReadDocumentTime", json_profile, 1); + _parse_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonParseTime", json_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonMaterializeTime", json_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonFilterTime", json_profile, 1); +} + Status JsonReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("JSON v2 reader requires scan params"); @@ -230,6 +250,7 @@ Status JsonReader::init(RuntimeState* state) { } Status JsonReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("JSON v2 file_schema is null"); } @@ -243,6 +264,8 @@ std::unique_ptr JsonReader::create_column_mapper( } Status JsonReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -269,6 +292,7 @@ Status JsonReader::open(std::shared_ptr request) { } Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -297,7 +321,10 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { bool is_empty_row = false; Status st = Status::OK(); try { - st = _parse_next_json(&size, &_reader_eof); + { + SCOPED_TIMER(_parse_time); + st = _parse_next_json(&size, &_reader_eof); + } if (st.ok() && !_reader_eof) { if (size == 0) { is_empty_row = true; @@ -306,6 +333,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } } if (st.ok() && !_reader_eof && !is_empty_row) { + SCOPED_TIMER(_materialize_time); st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); } } catch (simdjson::simdjson_error& e) { @@ -324,13 +352,17 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } *eof = _reader_eof && *rows == 0; _eof = *eof; return Status::OK(); } Status JsonReader::close() { + SCOPED_TIMER(_total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); @@ -489,6 +521,7 @@ Status JsonReader::_parse_jsonpath_and_json_root() { } Status JsonReader::_read_one_document(size_t* size, bool* eof) { + SCOPED_TIMER(_read_document_time); DORIS_CHECK(size != nullptr); DORIS_CHECK(eof != nullptr); *size = 0; diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index 52cdfad6728d64..de3c084e681f23 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -75,6 +75,7 @@ class JsonReader final : public FileReader { Status close() override; private: + void _init_profile() override; // A requested column keeps both identities: // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. // - `block_position`: index in the caller's output block, used for materialization. @@ -137,6 +138,13 @@ class JsonReader final : public FileReader { std::unordered_map _slot_name_to_index; std::vector _previous_positions; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_time = nullptr; + RuntimeProfile::Counter* _read_document_time = nullptr; + RuntimeProfile::Counter* _parse_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + io::FileReaderSPtr _physical_file_reader; std::unique_ptr _decompressor; std::unique_ptr _line_reader; diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 5d0984084a6d41..3d5429e0a9a262 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -29,6 +29,7 @@ #include "format_v2/materialized_reader_util.h" #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/slice.h" @@ -54,7 +55,26 @@ NativeReader::~NativeReader() { static_cast(close()); } +void NativeReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* native_profile = "NativeReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, native_profile, file_scan_profile::FILE_READER, 1); + _read_block_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeReadBlockTime", native_profile, 1); + _deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeDeserializeTime", native_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeMaterializeTime", native_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeFilterTime", native_profile, 1); +} + Status NativeReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_file_description == nullptr) { return Status::InvalidArgument("Native v2 reader requires file description"); @@ -65,6 +85,7 @@ Status NativeReader::init(RuntimeState* state) { } Status NativeReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("Native v2 file_schema is null"); } @@ -79,6 +100,7 @@ std::unique_ptr NativeReader::create_column_mapper( } Status NativeReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); _first_block_consumed = false; @@ -88,6 +110,7 @@ Status NativeReader::open(std::shared_ptr request) { } Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -108,6 +131,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { if (_first_block_loaded && !_first_block_consumed) { buffer = _first_block_buffer; } else { + SCOPED_TIMER(_read_block_time); RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); } @@ -131,11 +155,20 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { Block source_block; size_t uncompressed_bytes = 0; int64_t decompress_time = 0; - RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); - RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + { + SCOPED_TIMER(_deserialize_time); + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + } + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + } *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } if (_first_block_loaded && !_first_block_consumed) { _first_block_consumed = true; @@ -149,6 +182,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { } Status NativeReader::close() { + SCOPED_TIMER(_total_time); _file_reader.reset(); _tracing_file_reader.reset(); _request.reset(); diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h index 3719a6afd6c4f5..15a52fe6f8bc87 100644 --- a/be/src/format_v2/native/native_reader.h +++ b/be/src/format_v2/native/native_reader.h @@ -49,6 +49,7 @@ class NativeReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _validate_and_consume_header(); Status _ensure_schema_loaded() const; Status _read_next_pblock(std::string* buffer, bool* eof) const; @@ -65,6 +66,11 @@ class NativeReader final : public FileReader { mutable std::string _first_block_buffer; mutable bool _first_block_loaded = false; mutable bool _first_block_consumed = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_block_time = nullptr; + RuntimeProfile::Counter* _deserialize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; }; } // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index a5144d7b2017b0..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -27,6 +27,7 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "orc/Exceptions.hh" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "util/slice.h" @@ -54,24 +55,28 @@ class OrcMergedRangeFileReader final : public io::FileReader { _size(_file_reader->size()) { _statistics.apply_bytes += _range.end_offset - _range.start_offset; if (_profile != nullptr) { - const char* profile_name = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); - _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); - _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); - _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + const char* profile_name = "OrcMergedSmallIO"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); + // RuntimeProfile counter lookup is flat, so every child must be format-qualified; + // a unique parent alone cannot prevent aliasing with Parquet's MergeRange reader. + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedCopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestIO", TUnit::UNIT, profile_name, 1); - _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedIO", TUnit::UNIT, profile_name, 1); - _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, - profile_name, 1); - _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestBytes", + TUnit::BYTES, profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedBytes", TUnit::BYTES, profile_name, 1); - _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, - profile_name, 1); - _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, - profile_name, 1); - _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, - profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedApplyBytes", + TUnit::BYTES, profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedOverReadBytes", + TUnit::BYTES, profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedClusterNum", + TUnit::UNIT, profile_name, 1); } } @@ -113,6 +118,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { if (_profile == nullptr) { return; } + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -165,6 +171,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { OrcMergedRangeStatistics _statistics; RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; RuntimeProfile::Counter* _merged_io = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 9a2aeb538977dc..5248fcaf22dd11 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -76,6 +76,7 @@ #include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/segment/condition_cache.h" @@ -776,7 +777,9 @@ void OrcReader::_init_profile() { } static const char* orc_profile = "OrcReader"; - ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + file_scan_profile::ensure_hierarchy(_profile); + _orc_profile.total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -803,20 +806,22 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", - TUnit::UNIT, orc_profile, 1); + // RuntimeProfile counter names are flat; format-qualified names keep ORC ownership stable when + // one scan profile also initializes Parquet counters in either order. + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcRowGroupsFiltered", TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); - _orc_profile.read_row_groups = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", - TUnit::UNIT, orc_profile, 1); + _profile, "OrcRowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcRowGroupsReadNum", + TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcFilteredRowsByGroup", TUnit::UNIT, orc_profile, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_bytes = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); + _profile, "OrcFilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFilteredBytes", + TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFileNum", TUnit::UNIT, orc_profile, 1); } void OrcReader::_collect_profile() const { @@ -874,6 +879,8 @@ format::ColumnDefinition OrcReader::row_position_column_definition() { } Status OrcReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_orc_profile.total_time); RETURN_IF_ERROR(format::FileReader::init(state)); _state = std::make_unique(); TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); @@ -1162,6 +1169,7 @@ Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, } Status OrcReader::get_schema(std::vector* const file_schema) const { + SCOPED_TIMER(_orc_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -1195,6 +1203,7 @@ std::unique_ptr OrcReader::create_column_mapper( } Status OrcReader::open(std::shared_ptr request) { + SCOPED_TIMER(_orc_profile.total_time); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); } @@ -1877,6 +1886,7 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type } Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -1990,7 +2000,13 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { } _state->orc_lazy_selection_valid = false; } else { - RETURN_IF_ERROR(_filter_block(file_block, rows)); + auto filter_status = _filter_block(file_block, rows); + if (!filter_status.ok()) { + // V2 evaluates residual predicates after ORC returns the batch, while callers retain + // the historical nextBatch error contract used to identify row-filter failures. + filter_status.prepend("Orc row reader nextBatch failed. reason = "); + return filter_status; + } } *eof = false; return Status::OK(); @@ -2000,6 +2016,7 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { // NOLINTNEXTLINE(readability-function-size) Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); @@ -2455,6 +2472,7 @@ void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filt } Status OrcReader::close() { + SCOPED_TIMER(_orc_profile.total_time); _collect_profile(); if (_state != nullptr) { _state = std::make_unique(); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..a67dc5ad7f5e3e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -72,6 +72,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp new file mode 100644 index 00000000000000..713707b84b8e79 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -0,0 +1,870 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_desc.h" + +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "util/slice.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +static bool is_group_node(const tparquet::SchemaElement& schema) { + return schema.num_children > 0; +} + +static bool is_list_node(const tparquet::SchemaElement& schema) { + // Writers may emit only the modern logical type, so collection shape must not depend on the + // deprecated converted_type field being duplicated in the footer. + return (schema.__isset.converted_type && + schema.converted_type == tparquet::ConvertedType::LIST) || + (schema.__isset.logicalType && schema.logicalType.__isset.LIST); +} + +static bool is_map_node(const tparquet::SchemaElement& schema) { + return (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::MAP || + schema.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE)) || + (schema.__isset.logicalType && schema.logicalType.__isset.MAP); +} + +static bool has_primitive_only_annotation(const tparquet::SchemaElement& schema) { + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.DECIMAL || + logical.__isset.DATE || logical.__isset.TIME || logical.__isset.TIMESTAMP || + logical.__isset.INTEGER || logical.__isset.UNKNOWN || logical.__isset.JSON || + logical.__isset.BSON || logical.__isset.UUID || logical.__isset.FLOAT16 || + logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + return true; + } + } + if (!schema.__isset.converted_type) { + return false; + } + return schema.converted_type != tparquet::ConvertedType::MAP && + schema.converted_type != tparquet::ConvertedType::MAP_KEY_VALUE && + schema.converted_type != tparquet::ConvertedType::LIST; +} + +static bool is_repeated_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REPEATED; +} + +static bool is_required_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REQUIRED; +} + +static bool is_optional_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::OPTIONAL; +} + +static int num_children_node(const tparquet::SchemaElement& schema) { + return schema.__isset.num_children ? schema.num_children : 0; +} + +static Status validate_native_schema_structure( + const std::vector& schemas) { + if (schemas.empty()) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + const auto& root = schemas[0]; + if (root.__isset.type || !root.__isset.num_children || root.num_children < 0 || + (root.__isset.repetition_type && + root.repetition_type != tparquet::FieldRepetitionType::REQUIRED)) { + // Writers may encode the root's implicit REQUIRED repetition explicitly, and a zero-child + // root is a valid metadata-only schema. Optional or repeated roots remain ambiguous. + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + struct PendingGroup { + size_t remaining_children; + size_t depth; + }; + const auto root_children = num_children_node(schemas[0]); + if (root_children < 0 || static_cast(root_children) > schemas.size() - 1) { + return Status::InvalidArgument("Invalid parquet root child count {}", root_children); + } + std::vector pending {{static_cast(root_children), 0}}; + for (size_t pos = 1; pos < schemas.size(); ++pos) { + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (pending.empty()) { + return Status::InvalidArgument("Schema element {} is not reachable from the root", pos); + } + + const size_t depth = pending.back().depth + 1; + --pending.back().remaining_children; + const auto& schema = schemas[pos]; + if (!schema.__isset.repetition_type) { + return Status::InvalidArgument("Schema element {} has no repetition type", pos); + } + const bool has_children = schema.__isset.num_children && schema.num_children > 0; + if (schema.__isset.type == has_children) { + // Some legacy parquet-cpp files explicitly encode num_children=0 on primitive nodes. + // Only a positive count denotes a group, preserving strict rejection of real dual-kind + // nodes without dropping those otherwise valid files. + return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", + pos); + } + if (schema.__isset.num_children && schema.num_children < 0) { + return Status::InvalidArgument("Schema element {} has a negative child count", pos); + } + const int children = num_children_node(schemas[pos]); + if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { + return Status::InvalidArgument("Invalid child count {} at schema element {}", children, + pos); + } + if (children > 0) { + // Bound the tree before any resize or recursion so an untrusted footer cannot + // turn a tiny schema into an unbounded allocation or parser stack. + if (depth > MAX_NATIVE_SCHEMA_DEPTH) { + return Status::InvalidArgument("Parquet schema depth {} exceeds limit {}", depth, + MAX_NATIVE_SCHEMA_DEPTH); + } + pending.push_back({static_cast(children), depth}); + } + } + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (!pending.empty()) { + return Status::InvalidArgument("Parquet schema ended before all children were parsed"); + } + return Status::OK(); +} + +/** + * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. + * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the + * empty or null values in ancestor node. + * + * For instance, considering an array of strings with 3 rows like the following: + * null, [], [a, b, c] + * We can store four elements in data column: null, a, b, c + * and the offsets column is: 1, 1, 4 + * and the null map is: 1, 0, 0 + * For the i-th row in array column: range from `offsets[i - 1]` until `offsets[i]` represents the elements in this row, + * so we can't store empty array/map values in doris data column. + * As a comparison, spark does not require `repeated_parent_def_level`, + * because the spark column stores empty array/map values , and use anther length column to indicate empty values. + * Please reference: https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnVector.java + * + * Furthermore, we can also avoid store null array/map values in doris data column. + * The same three rows as above, We can only store three elements in data column: a, b, c + * and the offsets column is: 0, 0, 3 + * and the null map is: 1, 0, 0 + * + * Inherit the repetition and definition level from parent node, if the parent node is repeated, + * we should set repeated_parent_def_level = definition_level, otherwise as repeated_parent_def_level. + * @param parent parent node + * @param repeated_parent_def_level the first ancestor node whose repetition_type equals REPEATED + */ +static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_parent_def_level) { + for (auto& child : parent->children) { + child.repetition_level = parent->repetition_level; + child.definition_level = parent->definition_level; + child.repeated_parent_def_level = repeated_parent_def_level; + } +} + +static bool is_struct_list_node(const tparquet::SchemaElement& schema, + const std::string& enclosing_list_name) { + // The legacy Parquet exception is exact: accepting every "*_tuple" wrapper changes a standard + // one-child LIST wrapper from ARRAY to ARRAY>. + return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; +} + +static bool has_logical_annotation(const tparquet::SchemaElement& schema) { + return schema.__isset.logicalType || schema.__isset.converted_type; +} + +std::string NativeFieldSchema::debug_string() const { + std::stringstream ss; + ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level + << ", D=" << definition_level; + if (children.size() > 0) { + ss << ", type=" << data_type->get_name() << ", children=["; + for (int i = 0; i < children.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << children[i].debug_string(); + } + ss << "]"; + } else { + ss << ", physical_type=" << physical_type; + ss << " , doris_type=" << data_type->get_name(); + } + ss << ")"; + return ss.str(); +} + +Status NativeFieldDescriptor::parse_from_thrift( + const std::vector& t_schemas) { + _fields.clear(); + _physical_fields.clear(); + _name_to_field.clear(); + RETURN_IF_ERROR(validate_native_schema_structure(t_schemas)); + const auto& root_schema = t_schemas[0]; + _fields.resize(root_schema.num_children); + _next_schema_pos = 1; + + for (int i = 0; i < root_schema.num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &_fields[i])); + if (_name_to_field.find(_fields[i].name) != _name_to_field.end()) { + return Status::InvalidArgument("Duplicated field name: {}", _fields[i].name); + } + _name_to_field.emplace(_fields[i].name, &_fields[i]); + } + + if (_next_schema_pos != t_schemas.size()) { + return Status::InvalidArgument("Remaining {} unparsed schema elements", + t_schemas.size() - _next_schema_pos); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_node_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field) { + if (curr_pos >= t_schemas.size()) { + return Status::InvalidArgument("Out-of-bounds index of schema elements"); + } + auto& t_schema = t_schemas[curr_pos]; + if (is_group_node(t_schema)) { + // nested structure or nullable list + return parse_group_field(t_schemas, curr_pos, node_field); + } + if (is_repeated_node(t_schema)) { + // repeated (LIST) + // produce required list + node_field->repetition_level++; + node_field->definition_level++; + node_field->children.resize(1); + set_child_node_level(node_field, node_field->definition_level); + auto child = &node_field->children[0]; + parse_physical_field(t_schema, false, child); + + node_field->name = t_schema.name; + node_field->lower_case_name = to_lower(t_schema.name); + node_field->data_type = std::make_shared(make_nullable(child->data_type)); + _next_schema_pos = curr_pos + 1; + node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1; + } else { + bool is_optional = is_optional_node(t_schema); + if (is_optional) { + node_field->definition_level++; + } + parse_physical_field(t_schema, is_optional, node_field); + _next_schema_pos = curr_pos + 1; + } + return Status::OK(); +} + +void NativeFieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema, + bool is_nullable, + NativeFieldSchema* physical_field) { + physical_field->name = physical_schema.name; + physical_field->lower_case_name = to_lower(physical_field->name); + physical_field->parquet_schema = physical_schema; + physical_field->physical_type = physical_schema.type; + physical_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + _physical_fields.push_back(physical_field); + physical_field->physical_column_index = cast_set(_physical_fields.size() - 1); + std::pair type; + try { + type = get_doris_type(physical_schema, is_nullable); + } catch (const Exception& e) { + const bool is_decimal = + (physical_schema.__isset.logicalType && + physical_schema.logicalType.__isset.DECIMAL) || + (physical_schema.__isset.converted_type && + physical_schema.converted_type == tparquet::ConvertedType::DECIMAL); + // DECIMAL conversion failures carry precision/scale semantics that must not be erased by + // a raw-byte fallback. Other unknown annotations intentionally retain the established + // physical fallback, including legacy INTERVAL footers and forward-compatible types. + if (is_decimal) { + physical_field->unsupported_reason = e.what(); + } + auto fallback_schema = physical_schema; + fallback_schema.__isset.logicalType = false; + fallback_schema.__isset.converted_type = false; + type = get_doris_type(fallback_schema, is_nullable); + } + physical_field->data_type = type.first; + physical_field->is_type_compatibility = type.second; + physical_field->field_id = physical_schema.__isset.field_id ? physical_schema.field_id : -1; +} + +std::pair NativeFieldDescriptor::get_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + if (physical_schema.__isset.logicalType) { + ans = convert_to_doris_type(physical_schema.logicalType, nullable); + } else if (physical_schema.__isset.converted_type) { + ans = convert_to_doris_type(physical_schema, nullable); + } + if (ans.first->get_primitive_type() == PrimitiveType::INVALID_TYPE) { + switch (physical_schema.type) { + case tparquet::Type::BOOLEAN: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BOOLEAN, nullable); + break; + case tparquet::Type::INT32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::Type::INT64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::Type::INT96: + if (_enable_mapping_timestamp_tz) { + // treat INT96 as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, nullable, + 0, 6); + } else { + // in most cases, it's a nano timestamp + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, + 0, 6); + } + break; + case tparquet::Type::FLOAT: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + break; + case tparquet::Type::DOUBLE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, nullable); + break; + case tparquet::Type::BYTE_ARRAY: + if (_enable_mapping_varbinary) { + // if physical_schema not set logicalType and converted_type, + // we treat BYTE_ARRAY as VARBINARY by default, so that we can read all data directly. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet logicalType{}", + physical_schema.type)); + break; + } + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + tparquet::LogicalType logicalType, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + if (logicalType.__isset.STRING || logicalType.__isset.ENUM || logicalType.__isset.JSON || + logicalType.__isset.BSON) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.DECIMAL) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, + logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); + } else if (logicalType.__isset.DATE) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + } else if (logicalType.__isset.INTEGER) { + if (logicalType.INTEGER.isSigned) { + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } + } else { + is_type_compatibility = true; + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + } + } + } else if (logicalType.__isset.TIME) { + const int scale = logicalType.TIME.unit.__isset.MILLIS ? 3 : 6; + // TIME stores an integer unit, so its Doris scale must preserve the footer unit or + // sub-second values are silently truncated by the target SerDe. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); + } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + } else if (logicalType.__isset.UUID) { + if (_enable_mapping_varbinary) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, + -1, 16); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + } else if (logicalType.__isset.FLOAT16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else { + throw Exception(Status::InternalError("Not supported parquet logicalType")); + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + switch (physical_schema.converted_type) { + case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: + case tparquet::ConvertedType::type::JSON: + case tparquet::ConvertedType::type::BSON: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + case tparquet::ConvertedType::type::DECIMAL: + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DECIMAL128I, nullable, physical_schema.precision, physical_schema.scale); + break; + case tparquet::ConvertedType::type::DATE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + break; + case tparquet::ConvertedType::type::TIME_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIME_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::INT_8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_8: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_16: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_16: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::ConvertedType::type::UINT_32: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_64: + is_type_compatibility = true; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", + physical_schema.converted_type)); + } + return ans; +} + +Status NativeFieldDescriptor::parse_group_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field) { + auto& group_schema = t_schemas[curr_pos]; + if ((group_schema.__isset.logicalType && group_schema.logicalType.__isset.ENUM) || + (group_schema.__isset.converted_type && + group_schema.converted_type == tparquet::ConvertedType::ENUM)) { + // Preserve the established diagnostic used by compatibility tests and clients while the + // generic branch below handles every other primitive-only group annotation. + return Status::InvalidArgument("Logical type Enum cannot be applied to group node {}", + group_schema.name); + } + if (has_primitive_only_annotation(group_schema)) { + // Primitive annotations cannot change a group into a scalar. Reject them here instead of + // silently interpreting malformed metadata as an ordinary STRUCT. + return Status::InvalidArgument("Primitive annotation cannot be applied to group node {}", + group_schema.name); + } + if (is_map_node(group_schema)) { + // the map definition: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + return parse_map_field(t_schemas, curr_pos, group_field); + } + if (is_list_node(group_schema)) { + // the list definition: + // optional group (LIST) { + // repeated group [bag | list] { // hive or spark + // optional [array_element | element]; // hive or spark + // } + // } + return parse_list_field(t_schemas, curr_pos, group_field); + } + + if (is_repeated_node(group_schema)) { + group_field->repetition_level++; + group_field->definition_level++; + group_field->children.resize(1); + set_child_node_level(group_field, group_field->definition_level); + auto struct_field = &group_field->children[0]; + // the list of struct: + // repeated group (LIST) { + // optional/required ; + // ... + // } + // produce a non-null list + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field)); + + group_field->name = group_schema.name; + group_field->lower_case_name = to_lower(group_field->name); + group_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + group_field->data_type = + std::make_shared(make_nullable(struct_field->data_type)); + group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1; + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, group_field)); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_list_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, bool repeated_node_is_enclosing_list_element) { + // the list definition: + // spark and hive have three level schemas but with different schema name + // spark: - "list" - "element" + // hive: - "bag" - "array_element" + // parse three level schemas to two level primitive like: LIST, + // or nested structure like: LIST> + auto& first_level = t_schemas[curr_pos]; + if (first_level.num_children != 1) { + return Status::InvalidArgument("List element should have only one child"); + } + + if (curr_pos + 1 >= t_schemas.size()) { + return Status::InvalidArgument("List element should have the second level schema"); + } + + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED && + !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("List element can't be a repeated schema"); + } + + // the repeated schema element + auto& second_level = t_schemas[curr_pos + 1]; + if (second_level.repetition_type != tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("The second level of list element should be repeated"); + } + + // This indicates if this list is nullable. + bool is_optional = is_optional_node(first_level); + if (is_optional) { + list_field->definition_level++; + } + list_field->repetition_level++; + list_field->definition_level++; + list_field->children.resize(1); + NativeFieldSchema* list_child = &list_field->children[0]; + + size_t num_children = num_children_node(second_level); + if (num_children > 0) { + const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); + const auto& only_child = t_schemas[curr_pos + 2]; + if (num_children == 1 && !structural_wrapper && has_logical_annotation(second_level)) { + // The repeated node is already the outer LIST element. Preserve its own LIST/MAP + // annotation, but do not interpret its REPEATED marker as another outer array. + set_child_node_level(list_field, list_field->definition_level); + if (is_list_node(second_level)) { + RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); + } else if (is_map_node(second_level)) { + RETURN_IF_ERROR(parse_map_field(t_schemas, curr_pos + 1, list_child, true)); + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 1 && !structural_wrapper && !is_repeated_node(only_child)) { + // optional field, and the third level element is the nested structure in list + // produce nested structure like: LIST, LIST, LIST> + // skip bag/list, it's a repeated element. + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_node_field(t_schemas, curr_pos + 2, list_child)); + } else { + // required field, produce the list of struct + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 0) { + // required two level list, for compatibility reason. + set_child_node_level(list_field, list_field->definition_level); + parse_physical_field(second_level, false, list_child); + _next_schema_pos = curr_pos + 2; + } + + list_field->name = first_level.name; + list_field->lower_case_name = to_lower(first_level.name); + list_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + list_field->data_type = + std::make_shared(make_nullable(list_field->children[0].data_type)); + if (is_optional) { + list_field->data_type = make_nullable(list_field->data_type); + } + list_field->field_id = first_level.__isset.field_id ? first_level.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element) { + // the map definition in parquet: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + // Map value can be optional, the map without values is a SET + if (curr_pos + 2 >= t_schemas.size()) { + return Status::InvalidArgument("Map element should have at least three levels"); + } + auto& map_schema = t_schemas[curr_pos]; + if (map_schema.num_children != 1) { + return Status::InvalidArgument( + "Map element should have only one child(name='map', type='MAP_KEY_VALUE')"); + } + if (is_repeated_node(map_schema) && !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("Map element can't be a repeated schema"); + } + auto& map_key_value = t_schemas[curr_pos + 1]; + if (!is_group_node(map_key_value) || !is_repeated_node(map_key_value)) { + return Status::InvalidArgument( + "the second level in map must be a repeated group(key and value)"); + } + auto& map_key = t_schemas[curr_pos + 2]; + if (!is_required_node(map_key)) { + LOG(WARNING) << "Filed " << map_schema.name << " is map type, but with nullable key column"; + } + + if (map_key_value.num_children == 1) { + // The map with three levels is a SET + return parse_list_field(t_schemas, curr_pos, map_field, + repeated_node_is_enclosing_list_element); + } + if (map_key_value.num_children != 2) { + // A standard map should have four levels + return Status::InvalidArgument( + "the second level in map(MAP_KEY_VALUE) should have two children"); + } + // standard map + bool is_optional = is_optional_node(map_schema); + if (is_optional) { + map_field->definition_level++; + } + map_field->repetition_level++; + map_field->definition_level++; + + // Directly create key and value children instead of intermediate key_value node + map_field->children.resize(2); + // map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level` + set_child_node_level(map_field, map_field->definition_level); + + auto key_field = &map_field->children[0]; + auto value_field = &map_field->children[1]; + + // Parse key and value fields directly from the key_value group's children + _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field)); + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field)); + + map_field->name = map_schema.name; + map_field->lower_case_name = to_lower(map_field->name); + map_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + map_field->data_type = std::make_shared(make_nullable(key_field->data_type), + make_nullable(value_field->data_type)); + if (is_optional) { + map_field->data_type = make_nullable(map_field->data_type); + } + map_field->field_id = map_schema.__isset.field_id ? map_schema.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_struct_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* struct_field) { + // the nested column in parquet, parse group to struct. + auto& struct_schema = t_schemas[curr_pos]; + bool is_optional = is_optional_node(struct_schema); + if (is_optional) { + struct_field->definition_level++; + } + auto num_children = struct_schema.num_children; + struct_field->children.resize(num_children); + set_child_node_level(struct_field, struct_field->repeated_parent_def_level); + _next_schema_pos = curr_pos + 1; + for (int i = 0; i < num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i])); + } + struct_field->name = struct_schema.name; + struct_field->lower_case_name = to_lower(struct_field->name); + struct_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + + struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1; + DataTypes res_data_types; + std::vector names; + for (int i = 0; i < num_children; ++i) { + res_data_types.push_back(make_nullable(struct_field->children[i].data_type)); + names.push_back(struct_field->children[i].name); + } + struct_field->data_type = std::make_shared(res_data_types, names); + if (is_optional) { + struct_field->data_type = make_nullable(struct_field->data_type); + } + return Status::OK(); +} + +int NativeFieldDescriptor::get_column_index(const std::string& column) const { + for (int32_t i = 0; i < _fields.size(); i++) { + if (_fields[i].name == column) { + return i; + } + } + return -1; +} + +NativeFieldSchema* NativeFieldDescriptor::get_column(const std::string& name) const { + auto it = _name_to_field.find(name); + if (it != _name_to_field.end()) { + return it->second; + } + throw Exception(Status::InternalError("Name {} not found in NativeFieldDescriptor!", name)); + return nullptr; +} + +void NativeFieldDescriptor::get_column_names(std::unordered_set* names) const { + names->clear(); + for (const NativeFieldSchema& f : _fields) { + names->emplace(f.name); + } +} + +std::string NativeFieldDescriptor::debug_string() const { + std::stringstream ss; + ss << "fields=["; + for (int i = 0; i < _fields.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << _fields[i].debug_string(); + } + ss << "]"; + return ss.str(); +} + +void NativeFieldDescriptor::assign_ids() { + uint64_t next_id = 1; + for (auto& field : _fields) { + field.assign_ids(next_id); + } +} + +const NativeFieldSchema* NativeFieldDescriptor::find_column_by_id(uint64_t column_id) const { + for (const auto& field : _fields) { + if (auto result = field.find_column_by_id(column_id)) { + return result; + } + } + return nullptr; +} + +void NativeFieldSchema::assign_ids(uint64_t& next_id) { + column_id = next_id++; + + for (auto& child : children) { + child.assign_ids(next_id); + } + + max_column_id = next_id - 1; +} + +const NativeFieldSchema* NativeFieldSchema::find_column_by_id(uint64_t target_id) const { + if (column_id == target_id) { + return this; + } + + for (const auto& child : children) { + if (auto result = child.find_column_by_id(target_id)) { + return result; + } + } + + return nullptr; +} + +uint64_t NativeFieldSchema::get_column_id() const { + return column_id; +} + +void NativeFieldSchema::set_column_id(uint64_t id) { + column_id = id; +} + +uint64_t NativeFieldSchema::get_max_column_id() const { + return max_column_id; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h new file mode 100644 index 00000000000000..dfb669559fc71a --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nothing.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +// Constant for unassigned column IDs +constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; +constexpr size_t MAX_NATIVE_SCHEMA_DEPTH = 100; + +struct NativeFieldSchema { + std::string name; + std::string lower_case_name; // for hms column name case insensitive match + // the referenced parquet schema element + tparquet::SchemaElement parquet_schema; + + // Used to identify whether this field is a nested field. + DataTypePtr data_type; + // Schema construction keeps a physical fallback so unprojected columns and metadata-only + // queries remain readable, while projection validation reports the original logical failure. + std::string unsupported_reason; + + // Only valid when this field is a leaf node + tparquet::Type::type physical_type; + // The index order in NativeFieldDescriptor._physical_fields + int physical_column_index = -1; + int16_t definition_level = 0; + int16_t repetition_level = 0; + int16_t repeated_parent_def_level = 0; + std::vector children; + + //For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128. + bool is_type_compatibility = false; + + NativeFieldSchema() + : data_type(std::make_shared()), + column_id(NATIVE_UNASSIGNED_COLUMN_ID) {} + ~NativeFieldSchema() = default; + NativeFieldSchema(const NativeFieldSchema& fieldSchema) = default; + std::string debug_string() const; + + int32_t field_id = -1; + uint64_t column_id = NATIVE_UNASSIGNED_COLUMN_ID; + uint64_t max_column_id = 0; // Maximum column ID for this field and its children + + // Column ID assignment and lookup methods + void assign_ids(uint64_t& next_id); + const NativeFieldSchema* find_column_by_id(uint64_t target_id) const; + uint64_t get_column_id() const; + void set_column_id(uint64_t id); + uint64_t get_max_column_id() const; +}; + +// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. +class NativeFieldDescriptor { +private: + // Only the schema elements at the first level + std::vector _fields; + // The leaf node of schema elements + std::vector _physical_fields; + // Name to _fields, not all schema elements + std::unordered_map _name_to_field; + // Used in from_thrift, marking the next schema position that should be parsed + size_t _next_schema_pos; + // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type + bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; + +private: + void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, + NativeFieldSchema* physical_field); + + Status parse_list_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_struct_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* struct_field); + + Status parse_group_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field); + + Status parse_node_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field); + + std::pair convert_to_doris_type(tparquet::LogicalType logicalType, + bool nullable); + std::pair convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable); + std::pair get_doris_type(const tparquet::SchemaElement& physical_schema, + bool nullable); + +public: + NativeFieldDescriptor() = default; + ~NativeFieldDescriptor() = default; + + /** + * Parse NativeFieldDescriptor from parquet thrift FileMetaData. + * @param t_schemas list of schema elements + */ + Status parse_from_thrift(const std::vector& t_schemas); + + int get_column_index(const std::string& column) const; + + /** + * Get the column(the first level schema element, maybe nested field) by index. + * @param index Column index in _fields + */ + const NativeFieldSchema* get_column(size_t index) const { return &_fields[index]; } + + /** + * Get the column(the first level schema element, maybe nested field) by name. + * @param name Column name + * @return NativeFieldSchema or nullptr if not exists + */ + NativeFieldSchema* get_column(const std::string& name) const; + + void get_column_names(std::unordered_set* names) const; + + std::string debug_string() const; + + int32_t size() const { return cast_set(_fields.size()); } + + size_t physical_fields_size() const { return _physical_fields.size(); } + + const NativeFieldSchema* get_physical_field(size_t index) const { + return _physical_fields[index]; + } + + const std::vector& get_fields_schema() const { return _fields; } + + /** + * Assign stable column IDs to schema fields. + * + * This uses an ORC-compatible encoding so that the results of + * create_column_ids() are consistent across formats. IDs start from 1 + * and are assigned in a pre-order traversal (parent before children). + * After calling this, each NativeFieldSchema will have column_id and + * max_column_id populated. + */ + void assign_ids(); + + const NativeFieldSchema* find_column_by_id(uint64_t column_id) const; + void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp new file mode 100644 index 00000000000000..7539fb45efb46d --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_node.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +void NativeStructSchemaNode::add_child(std::string table_name, std::string file_name, + std::shared_ptr node) { + _children.emplace(std::move(table_name), Child {std::move(file_name), std::move(node)}); +} + +void NativeStructSchemaNode::add_missing_child(std::string table_name) { + _children.emplace(std::move(table_name), Child {}); +} + +std::shared_ptr NativeStructSchemaNode::child( + const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? nullptr : it->second.node; +} + +std::string NativeStructSchemaNode::file_child_name(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? std::string {} : it->second.file_name; +} + +bool NativeStructSchemaNode::has_child(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it != _children.end() && it->second.node != nullptr; +} + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result) { + if (projected_type == nullptr || result == nullptr) { + return Status::InvalidArgument("Native Parquet schema mapping input is null"); + } + const auto type = remove_nullable(projected_type); + switch (type->get_primitive_type()) { + case TYPE_STRUCT: { + if (file_schema.kind != ParquetColumnSchemaKind::STRUCT) { + return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); + } + const auto* struct_type = assert_cast(type.get()); + std::map file_children; + for (const auto& child : file_schema.children) { + file_children.emplace(to_lower(child->name), child.get()); + } + auto node = std::make_shared(); + for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { + const auto& table_name = struct_type->get_element_name(i); + // Native metadata keeps writer casing. Match normalized names while preserving the + // original file name used to address the physical child reader. + const auto child_it = file_children.find(to_lower(table_name)); + if (child_it == file_children.end()) { + node->add_missing_child(table_name); + continue; + } + std::shared_ptr child_node; + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *child_it->second, + &child_node)); + node->add_child(table_name, child_it->second->name, std::move(child_node)); + } + *result = std::move(node); + return Status::OK(); + } + case TYPE_ARRAY: { + if (file_schema.kind != ParquetColumnSchemaKind::LIST || file_schema.children.size() != 1) { + return Status::Corruption("Parquet column {} is not an ARRAY", file_schema.name); + } + const auto* array_type = assert_cast(type.get()); + std::shared_ptr element; + RETURN_IF_ERROR(build_native_schema_node(array_type->get_nested_type(), + *file_schema.children[0], &element)); + *result = std::make_shared(std::move(element)); + return Status::OK(); + } + case TYPE_MAP: { + if (file_schema.kind != ParquetColumnSchemaKind::MAP || file_schema.children.size() != 2) { + return Status::Corruption("Parquet column {} is not a MAP", file_schema.name); + } + const auto* map_type = assert_cast(type.get()); + std::shared_ptr key; + std::shared_ptr value; + RETURN_IF_ERROR( + build_native_schema_node(map_type->get_key_type(), *file_schema.children[0], &key)); + RETURN_IF_ERROR(build_native_schema_node(map_type->get_value_type(), + *file_schema.children[1], &value)); + *result = std::make_shared(std::move(key), std::move(value)); + return Status::OK(); + } + default: + if (file_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::Corruption("Parquet column {} is not a scalar", file_schema.name); + } + *result = std::make_shared(); + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.h b/be/src/format_v2/parquet/native_schema_node.h new file mode 100644 index 00000000000000..25c42ad4b2c6d4 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// V2-owned semantic mapping consumed by the native decoder. It deliberately contains no V1 +// reader/schema-helper state, so FileScannerV2 can build the mapping from its native metadata tree. +class NativeSchemaNode { +public: + virtual ~NativeSchemaNode() = default; + + virtual std::shared_ptr child(const std::string&) const { return nullptr; } + virtual std::string file_child_name(const std::string&) const { return {}; } + virtual bool has_child(const std::string&) const { return false; } + virtual std::shared_ptr element() const { return nullptr; } + virtual std::shared_ptr key() const { return nullptr; } + virtual std::shared_ptr value() const { return nullptr; } +}; + +class NativeScalarSchemaNode final : public NativeSchemaNode {}; + +class NativeStructSchemaNode final : public NativeSchemaNode { +public: + void add_child(std::string table_name, std::string file_name, + std::shared_ptr node); + void add_missing_child(std::string table_name); + + std::shared_ptr child(const std::string& table_name) const override; + std::string file_child_name(const std::string& table_name) const override; + bool has_child(const std::string& table_name) const override; + +private: + struct Child { + std::string file_name; + std::shared_ptr node; + }; + std::map _children; +}; + +class NativeArraySchemaNode final : public NativeSchemaNode { +public: + explicit NativeArraySchemaNode(std::shared_ptr element) + : _element(std::move(element)) {} + std::shared_ptr element() const override { return _element; } + +private: + std::shared_ptr _element; +}; + +class NativeMapSchemaNode final : public NativeSchemaNode { +public: + NativeMapSchemaNode(std::shared_ptr key, + std::shared_ptr value) + : _key(std::move(key)), _value(std::move(value)) {} + std::shared_ptr key() const override { return _key; } + std::shared_ptr value() const override { return _value; } + +private: + std::shared_ptr _key; + std::shared_ptr _value; +}; + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index 1cdfed80bd273b..7e541b57da2780 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -15,481 +15,215 @@ #include "format_v2/parquet/parquet_column_schema.h" -#include - #include #include #include #include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { namespace { -struct SchemaBuildContext { - int32_t local_id = -1; // child ordinal in the parent node - int16_t definition_level = 0; // accumulated optional/repeated level count - int16_t repetition_level = 0; // accumulated repeated level count - int16_t nullable_definition_level = 0; // definition level of the nearest optional node - int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node - int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node -}; - -enum class SchemaBuildMode { - // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for - // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected - // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct - // field boundary. - NORMAL, - // Build the current repeated node as the already-selected element of an enclosing LIST. This - // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the - // repeated node itself is the array element instead of a wrapper that should be stripped. - REPEATED_NODE_AS_LIST_ELEMENT, - // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP - // annotations on the repeated group itself. This keeps compatibility with the old Doris - // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". - REPEATED_NODE_AS_STRUCT_ELEMENT, -}; - -// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a -// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the -// element, or the element node itself. -struct ListElementResolution { - // Parquet node that should be exposed as Doris ARRAY element. - const ::parquet::schema::Node* element_node = nullptr; - // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state - // to materialize offsets, empty arrays and null arrays. - SchemaBuildContext repeated_context; - // Level state used to build element_node. This equals repeated_context when the repeated child - // itself is the element, and includes the wrapper's only child when standard 3-level LIST - // encoding is stripped. - SchemaBuildContext element_context; - // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, - // and the repeated level must not be interpreted as a second unrelated array at the same - // boundary. - SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; -}; - -// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet -// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] -// children. -struct MapEntryResolution { - const ::parquet::schema::GroupNode* entry_group = nullptr; - // Level state after consuming the repeated entry group. The parent MAP schema keeps this state - // to materialize offsets, empty maps and null maps. - SchemaBuildContext entry_context; -}; - -bool is_list_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::LIST || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); -} - -bool is_map_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::MAP || - node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); -} - -bool has_logical_annotation(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return (node.converted_type() != ::parquet::ConvertedType::NONE && - node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || - (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); -} - -bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { - return repeated_name == "array" || repeated_name == list_name + "_tuple"; -} - -bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { - return node.is_repeated() && !is_list_node(node) && !is_map_node(node); -} - -DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { - return node.is_optional() ? make_nullable(type) : type; -} - -void inherit_common_schema_state(const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - column_schema->local_id = context.local_id; - column_schema->parquet_field_id = node.field_id(); - column_schema->name = node.name(); - column_schema->max_definition_level = context.definition_level; - column_schema->max_repetition_level = context.repetition_level; - column_schema->nullable_definition_level = context.nullable_definition_level; - column_schema->definition_level = context.definition_level; - column_schema->repetition_level = context.repetition_level; - column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; - column_schema->repeated_repetition_level = context.repeated_repetition_level; -} - -SchemaBuildContext child_context(const SchemaBuildContext& parent, - const ::parquet::schema::Node& child_node, int32_t child_idx) { - SchemaBuildContext result = parent; - result.local_id = child_idx; - if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { - result.definition_level++; - result.nullable_definition_level = result.definition_level; - } - if (child_node.is_repeated()) { - result.repetition_level++; - result.definition_level++; - result.repeated_repetition_level = result.repetition_level; - result.repeated_ancestor_definition_level = result.definition_level; - } - return result; -} - -void propagate_child_levels(ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - for (const auto& child : column_schema->children) { - column_schema->max_definition_level = - std::max(column_schema->max_definition_level, child->max_definition_level); - column_schema->max_repetition_level = - std::max(column_schema->max_repetition_level, child->max_repetition_level); - } -} - -// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the -// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. -// Important cases: -// - repeated primitive: the primitive itself is the element (legacy two-level LIST). -// - repeated group with multiple children: the group itself is a STRUCT element. -// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per -// Parquet backward compatibility rules, even when it has one child or its own logical annotation. -// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. -// - other repeated group with a logical annotation, or whose only child is repeated: the group -// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct -// elements. -// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. -Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, - const SchemaBuildContext& list_context, - ListElementResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (list_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - const auto& repeated_node = *list_group.field(0); - if (!repeated_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - result->repeated_context = child_context(list_context, repeated_node, 0); - if (repeated_node.is_primitive()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& repeated_group = static_cast(repeated_node); - if (repeated_group.field_count() == 0) { - return Status::NotSupported("Unsupported parquet LIST element layout for column {}", - list_group.name()); +ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; } - const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); - if (repeated_group.field_count() > 1 || - has_structural_list_name(list_group.name(), repeated_group.name())) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; - return Status::OK(); + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; } - if (repeated_group_has_logical_annotation) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; } - - const auto& only_child = *repeated_group.field(0); - if (only_child.is_repeated()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - result->element_node = &only_child; - result->element_context = child_context(result->repeated_context, only_child, 0); - return Status::OK(); -} - -// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported -// two-level form in this reader: Doris requires a repeated group with exactly key and value -// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers -// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's -// definition levels and expose it as nullable for compatibility with the old reader. -Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, - const SchemaBuildContext& map_context, MapEntryResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (map_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - const auto& entry_node = *map_group.field(0); - if (!entry_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - if (entry_node.is_primitive()) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - const auto& entry_group = static_cast(entry_node); - if (entry_group.field_count() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files - // still mark the key field OPTIONAL even when all actual keys are non-null, for example: - // optional group t_map_varchar (MAP) { - // repeated group key_value { - // optional binary key (STRING); - // optional binary value (STRING); - // } - // } - // Accept that schema here so compatible files can be read. MapColumnReader validates the - // materialized key column and rejects data that really contains null map keys. - result->entry_group = &entry_group; - result->entry_context = child_context(map_context, entry_node, 0); - return Status::OK(); + return ParquetTimeUnit::UNKNOWN; } -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode); - -// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in -// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because -// protobuf and old parquet writers often encode repeated fields without a LIST annotation. -// Example: -// optional group event { -// repeated group links { -// optional binary url (UTF8); -// optional int32 rank; -// } -// } -// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's -// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. -// When the repeated field appears inside an already resolved LIST element, only the nested repeated -// child should be wrapped: -// optional group a (LIST) { -// repeated group element { -// repeated int32 items; -// } -// } -// The outer LIST element is the repeated "element" group, and its repeated "items" child should be -// represented as a field of type ARRAY inside the struct element. -Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& repeated_node, - const SchemaBuildContext& repeated_context, - std::unique_ptr* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); +ParquetExtraTypeInfo native_time_extra(ParquetTimeUnit unit) { + switch (unit) { + case ParquetTimeUnit::MILLIS: + return ParquetExtraTypeInfo::UNIT_MS; + case ParquetTimeUnit::MICROS: + return ParquetExtraTypeInfo::UNIT_MICROS; + case ParquetTimeUnit::NANOS: + return ParquetExtraTypeInfo::UNIT_NS; + case ParquetTimeUnit::UNKNOWN: + default: + return ParquetExtraTypeInfo::NONE; } - auto list_schema = std::make_unique(); - inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); - list_schema->kind = ParquetColumnSchemaKind::LIST; - list_schema->definition_level = repeated_context.definition_level; - list_schema->repetition_level = repeated_context.repetition_level; - list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; - - std::unique_ptr element_child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, - &element_child, - SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); - element_child->name = "element"; - list_schema->type = std::make_shared(element_child->type); - list_schema->children.push_back(std::move(element_child)); - propagate_child_levels(list_schema.get()); - *result = std::move(list_schema); - return Status::OK(); } -// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet -// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. -// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an -// enclosing LIST element, so only its nested bare repeated children should be wrapped. -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { - return build_repeated_field_as_list_schema(schema, node, context, result); - } - - auto column_schema = std::make_unique(); - inherit_common_schema_state(node, context, column_schema.get()); - - if (node.is_primitive()) { - const int leaf_column_id = schema.ColumnIndex(node); - if (leaf_column_id < 0) { - return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", - node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; - column_schema->leaf_column_id = leaf_column_id; - column_schema->descriptor = schema.Column(leaf_column_id); - if (column_schema->descriptor != nullptr) { - column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); - column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); +void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDescriptor* result) { + DORIS_CHECK(result != nullptr); + const auto& schema = field.parquet_schema; + result->doris_type = field.data_type; + result->unsupported_reason = field.unsupported_reason; + result->physical_type = static_cast(field.physical_type); + result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.DECIMAL) { + result->is_decimal = true; + result->decimal_precision = logical.DECIMAL.precision; + result->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.INTEGER) { + result->integer_bit_width = logical.INTEGER.bitWidth; + result->is_unsigned_integer = !logical.INTEGER.isSigned; + } else if (logical.__isset.TIME) { + result->time_unit = native_time_unit(logical.TIME.unit); + result->extra_type_info = native_time_extra(result->time_unit); + if (logical.TIME.isAdjustedToUTC) { + result->unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; + } + } else if (logical.__isset.TIMESTAMP) { + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + result->time_unit = native_time_unit(logical.TIMESTAMP.unit); + result->extra_type_info = native_time_extra(result->time_unit); + } else if (logical.__isset.FLOAT16) { + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; } - column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); - column_schema->type = column_schema->type_descriptor.doris_type; - if (column_schema->type == nullptr && - !column_schema->type_descriptor.unsupported_reason.empty()) { - // Keep unsupported logical leaves in the file schema using their physical storage - // type. For example, a file `{id: INT32, clock: TIME_MILLIS}` remains readable for - // `SELECT id`: schema mapping sees `clock` as its physical INT32 but never creates its - // reader. `SELECT clock` still fails explicitly in ParquetColumnReaderFactory before - // any physical value is decoded, preserving the unsupported-type contract. - column_schema->type = column_schema->type_descriptor.physical_doris_type; - } - if (column_schema->type == nullptr) { - return Status::NotSupported("Unsupported parquet column type for column {}", - node.name()); - } - column_schema->type = node.is_optional() - ? make_nullable(remove_nullable(column_schema->type)) - : remove_nullable(column_schema->type); - *result = std::move(column_schema); - return Status::OK(); - } - - const auto& group = static_cast(node); - if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); + } else if (schema.__isset.converted_type) { + switch (schema.converted_type) { + case tparquet::ConvertedType::DECIMAL: + result->is_decimal = true; + result->decimal_precision = schema.__isset.precision ? schema.precision : -1; + result->decimal_scale = schema.__isset.scale ? schema.scale : -1; + break; + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::UINT_8: + result->integer_bit_width = 8; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_8; + break; + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::UINT_16: + result->integer_bit_width = 16; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_16; + break; + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::UINT_32: + result->integer_bit_width = 32; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_32; + break; + case tparquet::ConvertedType::INT_64: + case tparquet::ConvertedType::UINT_64: + result->integer_bit_width = 64; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_64; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS + ? ParquetTimeUnit::MILLIS + : ParquetTimeUnit::MICROS; + result->extra_type_info = native_time_extra(result->time_unit); + break; + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::LIST; - ListElementResolution list_element; - RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); - column_schema->definition_level = list_element.repeated_context.definition_level; - column_schema->repetition_level = list_element.repeated_context.repetition_level; - column_schema->repeated_repetition_level = - list_element.repeated_context.repeated_repetition_level; - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, - list_element.element_context, &child, - list_element.element_build_mode)); - child->name = "element"; - column_schema->type = - nullable_if_needed(std::make_shared(child->type), node); - column_schema->children.push_back(std::move(child)); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } - if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); + if (result->is_decimal) { + switch (result->physical_type) { + case tparquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case tparquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::MAP; - MapEntryResolution map_entry; - RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); - column_schema->definition_level = map_entry.entry_context.definition_level; - column_schema->repetition_level = map_entry.entry_context.repetition_level; - column_schema->repeated_repetition_level = - map_entry.entry_context.repeated_repetition_level; - for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode( - schema, *map_entry.entry_group->field(child_idx), - child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), - child_idx), - &child, SchemaBuildMode::NORMAL)); - child->name = child_idx == 0 ? "key" : "value"; - column_schema->children.push_back(std::move(child)); - } - if (column_schema->children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - node.name()); - } - auto key_type = make_nullable(column_schema->children[0]->type); - auto value_type = make_nullable(column_schema->children[1]->type); - column_schema->type = - nullable_if_needed(std::make_shared(key_type, value_type), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } + } else if (result->physical_type == tparquet::Type::INT96) { + result->is_timestamp = true; + result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + result->is_string_like = !result->is_decimal && + result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result->physical_type == tparquet::Type::BYTE_ARRAY || + result->physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY); +} - column_schema->kind = ParquetColumnSchemaKind::STRUCT; - DataTypes child_types; - Strings child_names; - child_types.reserve(group.field_count()); - child_names.reserve(group.field_count()); - for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { - const auto& child_node = *group.field(child_idx); - std::unique_ptr child; - const auto child_ctx = child_context(context, child_node, child_idx); - if (should_build_repeated_field_as_list(child_node)) { - RETURN_IF_ERROR( - build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); - } else { - RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, - SchemaBuildMode::NORMAL)); - } - child_types.push_back(make_nullable(child->type)); - child_names.push_back(child->name); - column_schema->children.push_back(std::move(child)); +void propagate_native_max_levels(ParquetColumnSchema* schema) { + DORIS_CHECK(schema != nullptr); + for (const auto& child : schema->children) { + DORIS_CHECK(child != nullptr); + propagate_native_max_levels(child.get()); + schema->max_definition_level = + std::max(schema->max_definition_level, child->max_definition_level); + schema->max_repetition_level = + std::max(schema->max_repetition_level, child->max_repetition_level); } - column_schema->type = - nullable_if_needed(std::make_shared(child_types, child_names), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } -Status build_node_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, const SchemaBuildContext& context, - std::unique_ptr* result) { - return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); +std::unique_ptr build_native_node_schema(const NativeFieldSchema& field, + int32_t local_id) { + auto result = std::make_unique(); + result->local_id = local_id; + result->parquet_field_id = field.field_id; + result->name = field.name; + result->type = field.data_type; + result->definition_level = field.definition_level; + result->repetition_level = field.repetition_level; + result->max_definition_level = field.definition_level; + result->max_repetition_level = field.repetition_level; + result->nullable_definition_level = field.data_type != nullptr && field.data_type->is_nullable() + ? field.definition_level - field.repetition_level + : 0; + result->repeated_ancestor_definition_level = field.repeated_parent_def_level; + result->repeated_repetition_level = field.repetition_level; + + const auto primitive_type = remove_nullable(field.data_type)->get_primitive_type(); + if (field.children.empty()) { + result->kind = ParquetColumnSchemaKind::PRIMITIVE; + result->leaf_column_id = field.physical_column_index; + fill_native_type_descriptor(field, &result->type_descriptor); + return result; + } + if (primitive_type == TYPE_ARRAY) { + result->kind = ParquetColumnSchemaKind::LIST; + } else if (primitive_type == TYPE_MAP) { + result->kind = ParquetColumnSchemaKind::MAP; + } else { + result->kind = ParquetColumnSchemaKind::STRUCT; + } + result->children.reserve(field.children.size()); + for (size_t child_idx = 0; child_idx < field.children.size(); ++child_idx) { + result->children.push_back( + build_native_node_schema(field.children[child_idx], cast_set(child_idx))); + } + propagate_native_max_levels(result.get()); + return result; } } // namespace -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields) { if (fields == nullptr) { return Status::InvalidArgument("fields is null"); } fields->clear(); - const auto* root = schema.group_node(); - if (root == nullptr) { - return Status::InvalidArgument("Parquet schema root is null"); - } - fields->reserve(root->field_count()); - for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { - std::unique_ptr field; - SchemaBuildContext context; - RETURN_IF_ERROR(build_node_schema( - schema, *root->field(field_idx), - child_context(context, *root->field(field_idx), field_idx), &field)); - fields->push_back(std::move(field)); + const auto& native_fields = schema.get_fields_schema(); + fields->reserve(native_fields.size()); + for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { + // Unsupported logical leaves stay in the file schema so request-level validation can + // ignore unprojected fields and COUNT(*) placeholders without weakening real projections. + // The scan projection and native readers must share one tree; rebuilding wrappers through + // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. + fields->push_back( + build_native_node_schema(native_fields[field_idx], cast_set(field_idx))); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 1fb7262aabde6f..697bcd498382b1 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -23,18 +23,15 @@ #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_type.h" -namespace parquet { -class ColumnDescriptor; -class SchemaDescriptor; -} // namespace parquet - namespace doris::format::parquet { +class NativeFieldDescriptor; + enum class ParquetColumnSchemaKind { - PRIMITIVE, // primitive leaf -> ScalarColumnReader - STRUCT, // struct -> StructColumnReader - LIST, // array -> ListColumnReader - MAP, // map -> MapColumnReader + PRIMITIVE, // physical primitive leaf + STRUCT, // Parquet group with STRUCT semantics + LIST, // Parquet group with LIST semantics + MAP, // Parquet group with MAP semantics }; // ============================================================================ @@ -55,8 +52,6 @@ struct ParquetColumnSchema { ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; - const ::parquet::ColumnDescriptor* descriptor = nullptr; - // ======== Dremel Levels ======== int16_t max_definition_level = 0; @@ -74,7 +69,7 @@ struct ParquetColumnSchema { std::vector> children {}; }; -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index aa8f2622ade43b..c178e491b8ac21 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -15,154 +15,139 @@ #include "format_v2/parquet/parquet_file_context.h" -#include -#include #include -#include -#include #include #include #include #include -#include #include #include +#include "common/cast_set.h" #include "common/check.h" #include "common/config.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" -#include "storage/cache/page_cache.h" +#include "runtime/exec_env.h" +#include "util/coding.h" #include "util/slice.h" +#include "util/thrift_util.h" +#include "util/time.h" namespace doris::format::parquet { -namespace detail { - -namespace { - -bool page_cache_range_less(const ParquetPageCacheRange& lhs, const ParquetPageCacheRange& rhs) { - return lhs.offset < rhs.offset || (lhs.offset == rhs.offset && lhs.size < rhs.size); -} +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; -} // namespace - -ParquetPageCacheRangeIndex::ParquetPageCacheRangeIndex(size_t max_ranges) - : _max_ranges(max_ranges) { - DORIS_CHECK(_max_ranges > 0); +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) + : _metadata(std::move(metadata)), _parsed_size(parsed_size) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); } -void ParquetPageCacheRangeIndex::insert(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - return; - } - if (_ranges.size() >= _max_ranges) { - _ranges.erase(_ranges.begin()); - it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - } - _ranges.insert(it, range); +NativeParquetMetadata::~NativeParquetMetadata() { + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size()); } -void ParquetPageCacheRangeIndex::erase(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - const auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - _ranges.erase(it); +Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Native readers address projected leaves by stable DFS IDs. Assign them only on the private + // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. + _schema.assign_ids(); + for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { + const auto& row_group = _metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.columns.size() != _schema.physical_fields_size()) { + // All v2 planners index chunks by the native DFS leaf order, so validate cardinality + // once before any projection, prefetch, or decoder can perform indexed access. + return Status::Corruption( + "Parquet row group {} has {} column chunks but schema has {} physical fields", + row_group_idx, row_group.columns.size(), _schema.physical_fields_size()); + } + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + const auto* physical_field = _schema.get_physical_field(column_idx); + if (chunk.meta_data.type != physical_field->physical_type) { + return Status::Corruption( + "Parquet row group {} column {} physical type {} does not match schema {}", + row_group_idx, column_idx, tparquet::to_string(chunk.meta_data.type), + tparquet::to_string(physical_field->physical_type)); + } + if (chunk.meta_data.num_values < 0) { + return Status::Corruption( + "Parquet row group {} column {} has negative value count {}", row_group_idx, + column_idx, chunk.meta_data.num_values); + } + if (physical_field->repetition_level == 0 && + chunk.meta_data.num_values != row_group.num_rows) { + // A flat leaf contributes one definition-level slot per row, including NULLs. + // Reject a mismatched footer before metadata COUNT can treat it as cardinality. + return Status::Corruption( + "Parquet row group {} flat column {} has {} values but {} rows", + row_group_idx, column_idx, chunk.meta_data.num_values, row_group.num_rows); + } + } } + return Status::OK(); } -std::vector ParquetPageCacheRangeIndex::ranges() const { - std::lock_guard lock(_mutex); - return _ranges; -} - -size_t ParquetPageCacheRangeIndex::size() const { - std::lock_guard lock(_mutex); - return _ranges.size(); -} - -ParquetPageCacheRangeDirectory::ParquetPageCacheRangeDirectory(size_t max_files) - : _max_files(max_files) { - DORIS_CHECK(_max_files > 0); -} +namespace detail { -std::shared_ptr ParquetPageCacheRangeDirectory::get_or_create( - const std::string& file_key) { - DORIS_CHECK(!file_key.empty()); - std::lock_guard lock(_mutex); - if (const auto it = _indexes.find(file_key); it != _indexes.end()) { - return it->second; +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit) { + if (file_size < V2_PARQUET_FOOTER_SIZE || + serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); } - if (_indexes.size() >= _max_files) { - _indexes.erase(_indexes.begin()); + if (serialized_size > metadata_size_limit) { + return Status::Corruption("Parquet v2 footer size {} exceeds metadata limit {}", + serialized_size, metadata_size_limit); } - auto index = std::make_shared(); - _indexes.emplace(file_key, index); - return index; -} - -size_t ParquetPageCacheRangeDirectory::size() const { - std::lock_guard lock(_mutex); - return _indexes.size(); + return Status::OK(); } -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges) { - if (position < 0 || nbytes <= 0) { +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable) { + const int64_t mtime = description_mtime != 0 ? description_mtime : reader_mtime; + if (mtime == 0 && !is_immutable) { + // Unknown version is not a stable identity: an overwrite can preserve path and size while + // changing both footer semantics and page bytes. return {}; } + const int64_t file_size = description_file_size >= 0 ? description_file_size : reader_file_size; + return fmt::format("{}::{}::mtime={}::size={}", fs_name, path, mtime, file_size); +} - std::vector ranges; - ranges.reserve(cached_ranges.size()); - const int64_t request_end = position + nbytes; - for (const auto& range : cached_ranges) { - if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { - ranges.push_back(range); - } - } - std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { - if (lhs.offset != rhs.offset) { - return lhs.offset < rhs.offset; - } - return lhs.size > rhs.size; - }); - - std::vector plan; - int64_t cursor = position; - while (cursor < request_end) { - // At each cursor position, choose the cached range that already covers the cursor and - // extends farthest to the right. This handles both adjacent ranges and overlapping - // ranges. If no range covers the current cursor, there is a gap and the request must - // miss as a whole. - auto best = ranges.end(); - int64_t best_end = cursor; - for (auto it = ranges.begin(); it != ranges.end(); ++it) { - const int64_t cached_end = it->end_offset(); - if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { - best = it; - best_end = cached_end; - } - } - if (best == ranges.end()) { - return {}; - } - const int64_t copy_size = std::min(best_end, request_end) - cursor; - ParquetPageCacheReadPlanEntry entry; - entry.cached_range = *best; - entry.copy_offset_in_cache = cursor - best->offset; - entry.output_offset = cursor - position; - entry.copy_size = copy_size; - plan.push_back(entry); - cursor += copy_size; +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || + static_cast(offset) > file_size) { + return false; } - return plan; + return static_cast(length) <= file_size - static_cast(offset); +} + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { + return span_offset >= 0 && span_end >= span_offset && + span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; } std::vector valid_prefetch_ranges( @@ -197,455 +182,426 @@ bool should_use_merge_range_reader(const std::vector& ran avg_io_size < io::MergeRangeFileReader::SMALL_IO; } +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size) { + return file_size <= in_memory_file_size && + (path.starts_with("http://") || path.starts_with("https://")); +} + } // namespace detail namespace { -detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { - // Directory lookup is paid once when a reader opens. ReadAt() then synchronizes only on the - // shared index for this file, so unrelated Parquet files no longer serialize on a process-wide - // hot-path mutex. Strong references deliberately keep range discovery alive after reader A - // closes: reader B can reuse cached [100, 200) for a request [120, 150). The directory and each - // per-file index are independently capped, bounding stale metadata left by page-cache eviction. - static detail::ParquetPageCacheRangeDirectory directory; - return directory; +constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; +constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; + +Status parse_native_parquet_footer(io::FileReaderSPtr file, + std::unique_ptr* metadata, + size_t* footer_size, io::IOContext* io_ctx, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(footer_size != nullptr); + const size_t file_size = file->size(); + if (file_size < V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 file is too small for a footer: {}", file_size); + } + + const size_t tail_size = std::min(file_size, V2_INITIAL_FOOTER_READ_SIZE); + std::vector tail(tail_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - tail_size, Slice(tail.data(), tail.size()), + &bytes_read, io_ctx)); + if (bytes_read != tail.size()) { + return Status::Corruption("Short Parquet v2 footer read: expected {}, got {}", tail.size(), + bytes_read); + } + const auto* magic = tail.data() + tail.size() - sizeof(V2_PARQUET_MAGIC); + if (memcmp(magic, V2_PARQUET_MAGIC, sizeof(V2_PARQUET_MAGIC)) != 0) { + return Status::Corruption("Invalid Parquet v2 footer magic in {}", file->path().native()); + } + + const uint32_t serialized_size = + decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); + // The configured Thrift message ceiling also bounds this file-controlled allocation. Keep the + // check before both allocation and the optional second read so a sparse file cannot force a + // process-sized metadata buffer merely by advertising a large footer. + const size_t metadata_size_limit = + static_cast(std::max(config::thrift_max_message_size, 0)); + RETURN_IF_ERROR( + detail::validate_native_footer_size(serialized_size, file_size, metadata_size_limit)); + std::vector serialized_metadata(serialized_size); + if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { + const auto* metadata_start = + tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE - serialized_size; + memcpy(serialized_metadata.data(), metadata_start, serialized_size); + } else { + bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - V2_PARQUET_FOOTER_SIZE - serialized_size, + Slice(serialized_metadata.data(), serialized_metadata.size()), + &bytes_read, io_ctx)); + if (bytes_read != serialized_metadata.size()) { + return Status::Corruption("Short Parquet v2 metadata read: expected {}, got {}", + serialized_metadata.size(), bytes_read); + } + } + + uint32_t thrift_size = serialized_size; + tparquet::FileMetaData thrift_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, + &thrift_metadata)); + auto parsed = + std::make_unique(std::move(thrift_metadata), serialized_size); + RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); + *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; + *metadata = std::move(parsed); + return Status::OK(); } std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { - const int64_t mtime = - file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); - if (mtime == 0 && !file_description.is_immutable) { - // mtime == 0 means "unknown version", not the Unix epoch. V1 historically caches such a - // file under path::0, but copying that behavior for every V2 file is unsafe: a mutable file - // can be overwritten with different bytes while retaining both its path and size, causing - // process-global page cache entries to return stale data. Only callers that explicitly - // guarantee path immutability may use the mtime=0 cache key below. - return {}; - } - const int64_t file_size = file_description.file_size >= 0 - ? file_description.file_size - : static_cast(file_reader.size()); - return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, - file_reader.path().native(), mtime, file_size); + return detail::build_native_file_cache_key( + file_description.fs_name, file_reader.path().native(), file_description.mtime, + file_reader.mtime(), file_description.file_size, + static_cast(file_reader.size()), file_description.is_immutable); } -class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { -public: - DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) - : _file_reader(std::move(file_reader)), - _base_file_reader(_file_reader), - _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)), - _cached_page_range_index( - _enable_page_cache && !_page_cache_file_key.empty() - ? cached_page_range_directory().get_or_create(_page_cache_file_key) - : nullptr) { - DORIS_CHECK(_file_reader != nullptr); - if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { - _file_reader_stats = tracing_reader->stats(); - _base_file_reader = tracing_reader->inner_reader(); - } - DORIS_CHECK(_base_file_reader != nullptr); - set_mode(arrow::io::FileMode::READ); - } - - arrow::Status Close() override { - if (!_closed) { - collect_active_merge_range_profile(); - std::lock_guard lock(_page_cache_mutex); - // Page payloads and their bounded per-file range index intentionally outlive this - // reader for warm scans. Only reader-specific projected ranges are released here. - std::vector().swap(_page_cache_ranges); - _closed = true; - } - return arrow::Status::OK(); - } - - bool closed() const override { return _closed; } - - arrow::Result Tell() const override { return _pos; } - - arrow::Status Seek(int64_t position) override { - if (position < 0) { - return arrow::Status::Invalid("negative seek position"); - } - _pos = position; - return arrow::Status::OK(); - } - - arrow::Result GetSize() override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - return static_cast(_file_reader->size()); - } - - arrow::Result Read(int64_t nbytes, void* out) override { - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); - _pos += bytes_read; - return bytes_read; - } - - arrow::Result> Read(int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } +} // namespace - arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - if (position < 0 || nbytes < 0) { - return arrow::Status::Invalid("negative read position or length"); - } - if (try_read_from_page_cache(position, nbytes, out)) { - return nbytes; - } - size_t bytes_read = 0; - Status st = _file_reader->read_at( - static_cast(position), - Slice(static_cast(out), static_cast(nbytes)), &bytes_read, - _io_ctx); - if (!st.ok()) { - return arrow::Status::IOError(st.to_string_no_stack()); - } - insert_page_cache(position, nbytes, out, bytes_read); - return static_cast(bytes_read); - } +Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, + bool enable_page_cache, const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { + DORIS_CHECK(input_file_reader != nullptr); + if (detail::should_stage_small_http_file(input_file_reader->path().native(), + input_file_reader->size(), + config::in_memory_file_size)) { + // A metadata-cache hit can make the first physical read start inside a tiny HTTP file. + // Read it from byte zero once so EOF-range quirks cannot make warm scans less reliable + // than cold scans, while keeping this compatibility policy entirely inside v2. + native_file = std::make_shared(std::move(input_file_reader)); + } else { + native_file = std::move(input_file_reader); + } + native_io_ctx = io_ctx; + + // Footer and page bytes must use the same stable identity. In particular, fs_name separates + // identical HDFS paths from different nameservices, while an unknown mutable version bypasses + // both caches rather than reusing an overwritten file of the same size. + auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + auto meta_cache_key = build_page_cache_file_key(*native_file, file_description); + const bool has_stable_meta_cache_identity = !meta_cache_key.empty(); + if (has_stable_meta_cache_identity) { + // The discriminator prevents a v1 metadata value from being cast as the v2-owned type; + // schema mapping flags participate because they change the cached native schema tree. + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + } + size_t native_footer_size = 0; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { + native_metadata = native_meta_cache_handle.data(); + ++native_footer_cache_hits; + } else { + RETURN_IF_ERROR(parse_native_parquet_footer( + native_file, &native_metadata_owner, &native_footer_size, io_ctx, + enable_mapping_varbinary, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, native_metadata_owner.release(), + &native_meta_cache_handle); + native_metadata = native_meta_cache_handle.data(); + } else { + native_metadata = native_metadata_owner.get(); + } + } + DORIS_CHECK(native_metadata != nullptr); + + auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); + native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + // Native page readers use the FileDescription-derived immutable identity directly. + native_page_cache_file_key = page_cache_file_key; + return Status::OK(); +} - arrow::Result> ReadAt(int64_t position, - int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; +Status ParquetFileContext::load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const { + DORIS_CHECK(offset_indexes != nullptr); + offset_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - void register_page_cache_ranges(std::vector ranges) { - std::lock_guard lock(_page_cache_mutex); - _page_cache_ranges = std::move(ranges); + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for OffsetIndex", row_group_id); } - - void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - auto cached_reader = cached_remote_file_reader(); - if (cached_reader == nullptr) { - return; - } - const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; - for (const auto& range : ranges) { - if (range.offset < 0 || range.size <= 0) { + const auto& native_row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + try { + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(native_row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", + leaf_column_id); + } + const auto& column_chunk = native_row_group.columns[leaf_column_id]; + if (!column_chunk.__isset.offset_index_offset || + !column_chunk.__isset.offset_index_length || + column_chunk.offset_index_length <= 0) { continue; } - cached_reader->prefetch_range(static_cast(range.offset), - static_cast(range.size), prefetch_io_ctx); - } - } - - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - reset_active_file_reader(); - const auto valid_ranges = detail::valid_prefetch_ranges(ranges); - if (!detail::should_use_merge_range_reader( - valid_ranges, avg_io_size, - typeid_cast(_base_file_reader.get()) != nullptr)) { - return false; - } - - std::vector random_access_ranges; - random_access_ranges.reserve(valid_ranges.size()); - for (const auto& range : valid_ranges) { - random_access_ranges.emplace_back(static_cast(range.offset), - static_cast(range.end_offset())); - } - - // This mirrors the v1 parquet reader: when projected column chunks in a row group are - // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, - // but every page read below this point sees MergeRangeFileReader instead of the raw remote - // reader, so adjacent small requests can be coalesced and served from merge buffers. - // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later - // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once - // and satisfy the following ReadAt calls from its boxes, reducing remote request count. - _merge_range_active = true; - set_active_file_reader(std::make_shared( - profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); - return true; - } - - void reset_random_access_ranges() { reset_active_file_reader(); } - - ParquetPageCacheStats page_cache_stats() const { - std::lock_guard lock(_page_cache_mutex); - return _page_cache_stats; - } - -private: - bool page_cache_enabled() const { - return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty() && - _cached_page_range_index != nullptr; - } - - bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { - if (nbytes <= 0) { - return false; - } - const int64_t end = position + nbytes; - for (const auto& range : _page_cache_ranges) { - const int64_t range_end = range.offset + range.size; - if (position >= range.offset && end <= range_end) { - return true; + const int64_t index_offset = column_chunk.offset_index_offset; + const int64_t index_length = column_chunk.offset_index_length; + if (!detail::is_serialized_index_range_safe(native_file->size(), index_offset, + index_length)) { + // OffsetIndex is optional. A malformed range must not allocate from untrusted + // footer values or redirect the native reader outside the file. + continue; + } + std::vector serialized_index(static_cast(index_length)); + Slice index_slice(serialized_index.data(), serialized_index.size()); + size_t bytes_read = 0; + if (!native_file->read_at(index_offset, index_slice, &bytes_read, native_io_ctx).ok() || + bytes_read != serialized_index.size()) { + continue; + } + uint32_t thrift_length = static_cast(serialized_index.size()); + tparquet::OffsetIndex native_index; + if (!deserialize_thrift_msg(serialized_index.data(), &thrift_length, true, + &native_index) + .ok() || + native_index.page_locations.empty()) { + continue; + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + native_row_group.columns[leaf_column_id].meta_data, native_file->size(), + compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index( + native_index, chunk_range, + native_row_group.columns[leaf_column_id].meta_data.data_page_offset, + native_row_group.num_rows)) { + // OffsetIndex is optional. Reject the complete index instead of letting one bad + // location redirect an indexed reader outside its owning column chunk. + continue; } + offset_indexes->emplace(leaf_column_id, std::move(native_index)); } - return false; + } catch (const std::exception&) { + // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the + // native reader conservatively falls back to sequential page traversal. + offset_indexes->clear(); } + return Status::OK(); +} - StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { - return StoragePageCache::CacheKey(_page_cache_file_key, - static_cast(position + nbytes), position); +Status ParquetFileContext::load_native_page_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, int64_t* read_time, + int64_t* parse_time) const { + DORIS_CHECK(page_indexes != nullptr); + page_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, - int64_t copy_size, void* out, int64_t output_offset) { - PageCacheHandle handle; - if (!StoragePageCache::instance()->lookup( - page_cache_key(cached_range.offset, cached_range.size), &handle, - segment_v2::DATA_PAGE)) { - _cached_page_range_index->erase(cached_range); - return false; + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for PageIndex", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + + struct SerializedIndexRange { + int leaf_column_id; + int64_t offset; + int64_t length; + }; + struct PendingPageIndex { + NativeParquetPageIndex indexes; + bool has_column_index = false; + bool has_offset_index = false; + }; + std::vector column_index_ranges; + std::vector offset_index_ranges; + std::unordered_map pending_indexes; + + auto valid_index_range = [&](int64_t offset, int64_t length) { + return detail::is_serialized_index_range_safe(native_file->size(), offset, length); + }; + + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for PageIndex", leaf_column_id); + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (!chunk.__isset.column_index_offset || !chunk.__isset.column_index_length || + !chunk.__isset.offset_index_offset || !chunk.__isset.offset_index_length) { + continue; } - Slice cached = handle.data(); - const int64_t cache_offset = copy_position - cached_range.offset; - DORIS_CHECK(cache_offset >= 0); - DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); - memcpy(static_cast(out) + output_offset, cached.data + cache_offset, - static_cast(copy_size)); - return true; - } - - bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read(position, nbytes, - _cached_page_range_index->ranges()); - if (plan.empty()) { - return false; + if (!valid_index_range(chunk.column_index_offset, chunk.column_index_length) || + !valid_index_range(chunk.offset_index_offset, chunk.offset_index_length)) { + continue; } - for (const auto& entry : plan) { - if (!copy_cached_range(entry.cached_range, - entry.cached_range.offset + entry.copy_offset_in_cache, - entry.copy_size, out, entry.output_offset)) { - return false; + column_index_ranges.push_back( + {leaf_column_id, chunk.column_index_offset, chunk.column_index_length}); + offset_index_ranges.push_back( + {leaf_column_id, chunk.offset_index_offset, chunk.offset_index_length}); + pending_indexes.try_emplace(leaf_column_id); + } + + auto read_coalesced_indexes = [&](std::vector* ranges, + bool column_index) { + std::sort(ranges->begin(), ranges->end(), + [](const auto& lhs, const auto& rhs) { return lhs.offset < rhs.offset; }); + size_t range_begin = 0; + while (range_begin < ranges->size()) { + size_t range_end = range_begin + 1; + int64_t span_end = (*ranges)[range_begin].offset + (*ranges)[range_begin].length; + while (range_end < ranges->size() && (*ranges)[range_end].offset <= span_end) { + span_end = std::max(span_end, + (*ranges)[range_end].offset + (*ranges)[range_end].length); + ++range_end; } - } - return true; - } - bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { - return false; - } - ++_page_cache_stats.read_count; - // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact - // StoragePageCache key matches. - // Fallback path: Arrow may read a different but related byte range on another scan. - // Examples: - // - Current request [120, 150) can be served from cached [100, 200) by copying the - // 30-byte subset starting at cached offset 20. - // - Current request [100, 260) can be served by stitching cached [100, 180) and - // [180, 260). If any middle span is missing, it is a miss and the file reader fills - // the whole request from storage. - if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, - 0) && - !try_read_from_cached_ranges(position, nbytes, out)) { - ++_page_cache_stats.miss_count; - return false; - } - ++_page_cache_stats.hit_count; - ++_page_cache_stats.compressed_hit_count; - return true; - } - - void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || - bytes_read != static_cast(nbytes)) { - return; + const int64_t span_offset = (*ranges)[range_begin].offset; + if (!detail::is_serialized_index_span_safe(span_offset, span_end)) { + // Optional indexes share one allocation per contiguous footer block. Skipping the + // whole block prevents many individually small ranges from bypassing the budget. + range_begin = range_end; + continue; + } + const int64_t span_length = span_end - span_offset; + std::vector serialized(static_cast(span_length)); + Slice slice(serialized.data(), serialized.size()); + size_t bytes_read = 0; + Status read_status; + int64_t read_time_sink = 0; + { + SCOPED_RAW_TIMER(read_time == nullptr ? &read_time_sink : read_time); + read_status = native_file->read_at(span_offset, slice, &bytes_read, native_io_ctx); + } + if (read_status.ok() && bytes_read == serialized.size()) { + for (size_t i = range_begin; i < range_end; ++i) { + const auto& range = (*ranges)[i]; + auto pending = pending_indexes.find(range.leaf_column_id); + if (pending == pending_indexes.end()) { + continue; + } + uint32_t thrift_length = static_cast(range.length); + const auto* thrift_data = + serialized.data() + static_cast(range.offset - span_offset); + int64_t parse_time_sink = 0; + SCOPED_RAW_TIMER(parse_time == nullptr ? &parse_time_sink : parse_time); + if (column_index) { + pending->second.has_column_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.column_index) + .ok(); + } else { + pending->second.has_offset_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.offset_index) + .ok(); + } + } + } + range_begin = range_end; } - auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); - memcpy(page->data(), data, bytes_read); - PageCacheHandle handle; - StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, - segment_v2::DATA_PAGE); - _cached_page_range_index->insert( - ParquetPageCacheRange {.offset = position, .size = nbytes}); - ++_page_cache_stats.write_count; - ++_page_cache_stats.compressed_write_count; - } - - void set_active_file_reader(io::FileReaderSPtr reader) { - DORIS_CHECK(reader != nullptr); - _file_reader = _file_reader_stats != nullptr - ? std::make_shared(std::move(reader), - _file_reader_stats) - : std::move(reader); - } - - void reset_active_file_reader() { - collect_active_merge_range_profile(); - _merge_range_active = false; - set_active_file_reader(_base_file_reader); - } + }; - void collect_active_merge_range_profile() { - if (_merge_range_active && _file_reader != nullptr) { - // MergeRangeFileReader writes its MergedSmallIO counters only from - // collect_profile_before_close(). v2 replaces the active reader for every row group, - // so collect before overwriting it; Close() handles the final row group. Example: - // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the - // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. - _file_reader->collect_profile_before_close(); - } - } + // Parquet writers place each index kind in a contiguous block. Reading overlapping/adjacent + // ranges as one span keeps cold small-file planning from paying two remote round trips per + // projected leaf, while refusing gaps avoids amplifying reads from untrusted footer offsets. + read_coalesced_indexes(&column_index_ranges, true); + read_coalesced_indexes(&offset_index_ranges, false); - std::shared_ptr cached_remote_file_reader() { - if (_merge_range_active) { - return nullptr; - } - auto reader = _file_reader; - if (reader == nullptr) { - return nullptr; + for (auto& [leaf_column_id, pending] : pending_indexes) { + const auto& chunk = row_group.columns[leaf_column_id]; + auto& indexes = pending.indexes; + if (!pending.has_column_index || !pending.has_offset_index || + indexes.column_index.null_pages.size() != indexes.offset_index.page_locations.size()) { + continue; } - // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are - // enabled. Prefetch should target the physical cached reader below that tracing wrapper, - // otherwise v2 scans with profiling would silently lose prefetch. - if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { - reader = tracing_reader->inner_reader(); + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(indexes.offset_index, chunk_range, + chunk.meta_data.data_page_offset, row_group.num_rows)) { + continue; } - return std::dynamic_pointer_cast(reader); + page_indexes->emplace(leaf_column_id, std::move(indexes)); } + return Status::OK(); +} - io::FileReaderSPtr _file_reader; - io::FileReaderSPtr _base_file_reader; - io::FileReaderStats* _file_reader_stats = nullptr; - io::IOContext* _io_ctx = nullptr; - int64_t _pos = 0; - bool _closed = false; - bool _enable_page_cache = false; - bool _merge_range_active = false; - std::string _page_cache_file_key; - mutable std::mutex _page_cache_mutex; - std::vector _page_cache_ranges; - std::shared_ptr _cached_page_range_index; - ParquetPageCacheStats _page_cache_stats; -}; - -} // namespace - -Status arrow_status_to_doris_status(const arrow::Status& status) { - if (status.ok()) { - return Status::OK(); +void ParquetFileContext::prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + io::FileReaderSPtr reader = native_file; + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); } - if (status.IsIOError()) { - return Status::IOError(status.ToString()); + auto cached_reader = std::dynamic_pointer_cast(reader); + if (cached_reader == nullptr) { + return; } - if (status.IsInvalid()) { - return Status::InvalidArgument(status.ToString()); + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : native_io_ctx; + for (const auto& range : detail::valid_prefetch_ranges(ranges)) { + cached_reader->prefetch_range(cast_set(range.offset), cast_set(range.size), + prefetch_io_ctx); } - return Status::InternalError(status.ToString()); } -Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { - DORIS_CHECK(input_file_reader != nullptr); - auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); - try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); - metadata = this->file_reader->metadata(); - schema = metadata != nullptr ? metadata->schema() : nullptr; - } catch (const ::parquet::ParquetException& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet file: {}", e.what()); - } catch (const std::exception& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet file: {}", e.what()); +bool ParquetFileContext::set_native_random_access_ranges( + const std::vector& ranges, size_t avg_io_size, + RuntimeProfile* profile, int64_t merge_read_slice_size) { + DORIS_CHECK(native_file != nullptr); + if (!detail::should_use_merge_range_reader( + ranges, avg_io_size, + typeid_cast(native_file.get()) != nullptr)) { + native_row_group_file = native_file; + return false; } - if (metadata == nullptr || schema == nullptr) { - return Status::Corruption("Failed to read parquet metadata"); + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + std::vector native_ranges; + native_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + native_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); } - return Status::OK(); -} - -void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get()) - ->register_page_cache_ranges(std::move(ranges)); -} - -void ParquetFileContext::prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); -} - -bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - DORIS_CHECK(arrow_file != nullptr); - return static_cast(arrow_file.get()) - ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); + std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); + native_row_group_file = std::make_shared( + profile, native_file, native_ranges, merge_read_slice_size); + return true; } void ParquetFileContext::reset_random_access_ranges() { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->reset_random_access_ranges(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { - if (arrow_file == nullptr) { - return {}; - } - return static_cast(arrow_file.get())->page_cache_stats(); + return {}; } Status ParquetFileContext::close() { - if (file_reader != nullptr) { - try { - file_reader->Close(); - } catch (const std::exception&) { - } - } - if (arrow_file != nullptr) { - static_cast(arrow_status_to_doris_status(arrow_file->Close())); - } - file_reader.reset(); - arrow_file.reset(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); + native_metadata = nullptr; + native_metadata_owner.reset(); + native_meta_cache_handle = {}; + native_file.reset(); + native_io_ctx = nullptr; + native_page_cache_enabled = false; + native_page_cache_file_key.clear(); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 67e9102139dbf2..0cd413e10557ac 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -15,19 +15,21 @@ #pragma once -#include -#include +#include #include #include #include -#include #include +#include #include +#include #include #include "common/status.h" +#include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" +#include "util/obj_lru_cache.h" namespace doris::io { struct FileDescription; @@ -40,6 +42,26 @@ class RuntimeProfile; namespace doris::format::parquet { +struct NativeParquetPageIndex; + +// V2-owned footer/schema tree. Production planning and decoding consume this object directly; +// Arrow metadata is intentionally not materialized from the serialized footer. +class NativeParquetMetadata { +public: + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size); + ~NativeParquetMetadata(); + + Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + const tparquet::FileMetaData& to_thrift() const { return _metadata; } + const NativeFieldDescriptor& schema() const { return _schema; } + size_t get_mem_size() const { return _parsed_size; } + +private: + tparquet::FileMetaData _metadata; + NativeFieldDescriptor _schema; + size_t _parsed_size = 0; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -47,17 +69,6 @@ struct ParquetPageCacheRange { int64_t end_offset() const { return offset + size; } }; -struct ParquetPageCacheReadPlanEntry { - // The exact cached StoragePageCache entry. The final cache key is still exact-range based: - // file key + cached_range.end_offset() + cached_range.offset. - ParquetPageCacheRange cached_range; - // Byte offset inside cached_range to start copying from. - int64_t copy_offset_in_cache = 0; - // Byte offset inside the current ReadAt output buffer to start writing to. - int64_t output_offset = 0; - int64_t copy_size = 0; -}; - struct ParquetPageCacheStats { int64_t read_count = 0; int64_t write_count = 0; @@ -69,54 +80,19 @@ struct ParquetPageCacheStats { namespace detail { -class ParquetPageCacheRangeIndex { -public: - static constexpr size_t DEFAULT_MAX_RANGES = 65536; - - explicit ParquetPageCacheRangeIndex(size_t max_ranges = DEFAULT_MAX_RANGES); +inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; - void insert(ParquetPageCacheRange range); - void erase(ParquetPageCacheRange range); +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit); - std::vector ranges() const; - size_t size() const; +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable); -private: - const size_t _max_ranges; - mutable std::mutex _mutex; - std::vector _ranges; -}; - -class ParquetPageCacheRangeDirectory { -public: - static constexpr size_t DEFAULT_MAX_FILES = 4096; - - explicit ParquetPageCacheRangeDirectory(size_t max_files = DEFAULT_MAX_FILES); +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); - std::shared_ptr get_or_create(const std::string& file_key); - size_t size() const; - -private: - const size_t _max_files; - mutable std::mutex _mutex; - std::unordered_map> _indexes; -}; - -// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of -// previously cached entries. -// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The -// caller therefore keeps lightweight cached range metadata and uses this function to decide -// which exact cache entries to fetch and which byte spans to copy. -// Examples: -// 1. Subset hit: -// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. -// 2. Superset hit covered by multiple cached entries: -// request [100, 260), cached [100, 180) and [180, 260) -// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. -// 3. Partial overlap is a miss: -// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges); +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); // Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is // expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted @@ -125,54 +101,74 @@ std::vector plan_page_cache_range_read( std::vector valid_prefetch_ranges( const std::vector& ranges); -// Average projected column-chunk size for one row group. The v1 parquet path uses this value to -// decide whether a row group is dominated by small random IOs; v2 uses the same signal before -// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are -// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +// Average projected column-chunk size for one row group. V2 uses this signal to decide whether the +// row group is dominated by small random IOs before installing MergeRangeFileReader. Example: +// chunks of 512KB and 1MB average below SMALL_IO and are good merge-reader candidates, while two +// 8MB chunks should stay on the raw random-access reader. size_t average_prefetch_range_size(const std::vector& ranges); -// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row -// group. This is intentionally stricter than the background warm-up path: +// Decide whether native data-page ReadAt() should be routed through MergeRangeFileReader for the +// current row group. This is intentionally stricter than the background warm-up path: // - no valid projected chunks -> nothing to merge; // - in-memory file readers already avoid remote random IO; // - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. bool should_use_merge_range_reader(const std::vector& ranges, size_t avg_io_size, bool is_in_memory_reader); +// HTTP range servers may reject an overlong range near EOF even after accepting the capability +// probe. Staging a bounded small HTTP object also turns all native page reads into memory copies. +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size); + } // namespace detail struct ParquetFileContext { - std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader - std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser - std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) - const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema + // Native metadata, index, and data-page paths share Doris' FileReader without transferring + // ownership to an external metadata tree. + io::FileReaderSPtr native_file; + // Row-group-scoped view of native_file. Small projected chunks use MergeRangeFileReader; + // large chunks and in-memory files keep native_file. + io::FileReaderSPtr native_row_group_file; + io::IOContext* native_io_ctx = nullptr; + // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is + // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + const NativeParquetMetadata* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; + ObjLRUCache::CacheHandle native_meta_cache_handle; + int64_t native_footer_read_calls = 0; + int64_t native_footer_cache_hits = 0; + bool native_page_cache_enabled = false; + std::string native_page_cache_file_key; Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); - // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page - // decoding, so v2 caches the serialized bytes read inside these ranges and excludes - // footer/metadata reads that happen before registration. - void register_page_cache_ranges(std::vector ranges); + const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + Status load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const; + Status load_native_page_indexes(int row_group_id, + const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, + int64_t* read_time = nullptr, + int64_t* parse_time = nullptr) const; // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current - // row group's projected column chunks are small random IOs. This is the real v1-compatible - // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely - // warming file cache in the background. Returns true when merge-range reading is active. - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); - // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader - // counters. Row-group setup uses this before dictionary-page probes, because those probes are - // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + // Install the row-group-scoped MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their native ReadAt order is independent of the + // sequential projected chunk ranges consumed by MergeRangeFileReader. + bool set_native_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + const io::FileReaderSPtr& native_data_file() const { + return native_row_group_file != nullptr ? native_row_group_file : native_file; + } + // Restore native ReadAt() to the base Doris file reader and flush merge-reader counters. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); }; -Status arrow_status_to_doris_status(const arrow::Status& status); - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 29783e14a05071..b56a57d419e0f3 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_statistics.h" +#include "runtime/file_scan_profile.h" namespace doris::format::parquet { @@ -26,9 +27,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { return; } + file_scan_profile::ensure_hierarchy(profile); static const char* parquet_profile = "ParquetReader"; - ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + total_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); + // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them + // below the format node so profile parsers and operators can attribute pruning to Parquet. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -55,10 +60,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); + // Keep every Parquet scan metric below the format node: profile consumers extract that + // subtree and otherwise silently lose this counter even though filtering happened. rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + dense_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); + selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, + parquet_profile, 1); empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", TUnit::UNIT, parquet_profile, 1); range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", @@ -69,14 +80,30 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, parquet_profile, 1); - arrow_read_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); - arrow_skip_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowSkipRecordsTime", parquet_profile, 1); + level_only_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); + level_only_skip_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); + native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, + parquet_profile, 1); + native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", + TUnit::UNIT, parquet_profile, 1); + page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", + TUnit::UNIT, parquet_profile, 1); + nested_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); + // Format-specific counters stay below ParquetReader so subtree consumers cannot silently + // attribute them to a different file format initialized on the same RuntimeProfile. filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1); raw_rows_read = @@ -88,7 +115,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); - page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, + parquet_profile, 1); page_index_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); read_page_index_time = @@ -103,8 +131,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); row_group_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); - file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); - file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", + TUnit::UNIT, parquet_profile, 1); + file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, + parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); @@ -139,6 +169,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + predicate_compaction_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); + predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", + TUnit::BYTES, parquet_profile, 1); + predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", + TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectRows", TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -157,7 +197,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); - convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); bloom_filter_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); } @@ -176,12 +215,42 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + +void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const { + const int64_t filtered = selected ? 0 : 1; + COUNTER_UPDATE(filtered_row_groups, filtered); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + // Initial footer planning counts every surviving candidate as readable. Lazy probes correct + // that estimate only when a later dictionary, Bloom, or page-index check removes the group. + COUNTER_UPDATE(to_read_row_groups, -filtered); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); @@ -199,9 +268,35 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .reader_read_rows = reader_read_rows, .reader_skip_rows = reader_skip_rows, .reader_select_rows = reader_select_rows, - .arrow_read_records_time = arrow_read_records_time, - .arrow_skip_records_time = arrow_skip_records_time, + .level_only_read_time = level_only_read_time, + .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, + .hybrid_selection_batches = hybrid_selection_batches, + .hybrid_selection_ranges = hybrid_selection_ranges, + .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, + .decompress_time = decompress_time, + .decompress_count = decompress_cnt, + .decode_header_time = decode_header_time, + .decode_value_time = decode_value_time, + .decode_dictionary_time = decode_dict_time, + .decode_level_time = decode_level_time, + .decode_null_map_time = decode_null_map_time, + .page_index_read_calls = page_index_read_calls, + .skip_page_header_count = skip_page_header_num, + .parse_page_header_count = parse_page_header_num, + .read_page_header_time = read_page_header_time, + .page_read_count = page_read_counter, + .page_cache_write_count = page_cache_write_counter, + .page_cache_compressed_write_count = page_cache_compressed_write_counter, + .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, + .page_cache_hit_count = page_cache_hit_counter, + .page_cache_miss_count = page_cache_missing_counter, + .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, + .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, + .native_read_calls = native_read_calls, + .native_page_fragments = native_page_fragments, + .page_crossing_batches = page_crossing_batches, + .nested_batches = nested_batches, }; } @@ -212,10 +307,17 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .rows_filtered_by_conjunct = rows_filtered_by_conjunct, .lazy_read_filtered_rows = lazy_read_filtered_rows, .total_batches = total_batches, + .dense_batches = dense_batches, + .selected_batches = selected_batches, .empty_selection_batches = empty_selection_batches, .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, .predicate_filter_time = predicate_filter_time, + .predicate_compaction_time = predicate_compaction_time, + .predicate_compaction_bytes = predicate_compaction_bytes, + .predicate_compaction_count = predicate_compaction_count, + .fixed_width_predicate_direct_batches = fixed_width_predicate_direct_batches, + .fixed_width_predicate_direct_rows = fixed_width_predicate_direct_rows, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 827cac45d94df0..170f14b56c5d14 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -31,28 +31,65 @@ struct ParquetPageSkipProfile { // ============================================================================ // ============================================================================ struct ParquetColumnReaderProfile { - RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() - RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() - RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; // Arrow SkipRecords time (ns) - RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + // COUNT(nullable_col) shape-only path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + // Native page/encoding reader internals. These counters keep page IO, decompression, levels, + // value decode and conversion attributable to separate stages. + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_count = nullptr; + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dictionary_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* skip_page_header_count = nullptr; + RuntimeProfile::Counter* parse_page_header_count = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* page_read_count = nullptr; + RuntimeProfile::Counter* page_cache_write_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_miss_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; // native column-reader calls + RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments + RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages + RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches }; // ============================================================================ // ============================================================================ struct ParquetScanProfile { - RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* raw_rows_read = nullptr; // logical rows consumed before filtering RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; // rows avoided by late materialization RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* dense_batches = nullptr; // batches retaining every physical row + RuntimeProfile::Counter* selected_batches = + nullptr; // non-empty batches compacted by predicates RuntimeProfile::Counter* empty_selection_batches = nullptr; // empty batches after full filtering RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -73,11 +110,15 @@ struct ParquetScanProfile { struct ParquetProfile { void init(RuntimeProfile* profile); void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + void update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const; ParquetPageSkipProfile page_skip_profile() const; ParquetColumnReaderProfile column_reader_profile() const; ParquetScanProfile scan_profile() const; + RuntimeProfile::Counter* total_time = nullptr; + RuntimeProfile::Counter* filtered_row_groups = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; @@ -96,6 +137,8 @@ struct ParquetProfile { RuntimeProfile::Counter* selected_rows = nullptr; RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* dense_batches = nullptr; + RuntimeProfile::Counter* selected_batches = nullptr; RuntimeProfile::Counter* empty_selection_batches = nullptr; RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; @@ -103,9 +146,17 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; - RuntimeProfile::Counter* arrow_read_records_time = nullptr; - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; + // COUNT(nullable_col) shape-only path; ordinary scans keep these zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; + RuntimeProfile::Counter* native_page_fragments = nullptr; + RuntimeProfile::Counter* page_crossing_batches = nullptr; + RuntimeProfile::Counter* nested_batches = nullptr; RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* filtered_bytes = nullptr; @@ -149,6 +200,11 @@ struct ParquetProfile { RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; @@ -158,7 +214,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index cd84ab74ef953e..4a80bc0403df8d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -39,7 +39,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" -#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/count_column_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -57,69 +57,6 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; -int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - -void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); - } - return; - } - for (const auto& child : column_schema.children) { - DORIS_CHECK(child != nullptr); - collect_all_leaf_column_ids(*child, leaf_column_ids); - } -} - -void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (projection.project_all_children || projection.children.empty()) { - collect_all_leaf_column_ids(column_schema, leaf_column_ids); - return; - } - for (const auto& child_projection : projection.children) { - const auto child_it = - std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { - return child_schema->local_id == child_projection.local_id(); - }); - DORIS_CHECK(child_it != column_schema.children.end()); - collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); - } -} - -void collect_request_leaf_column_ids( - const std::vector>& file_schema, - const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { - const auto local_id = projection.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID || - local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return; - } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); - }; - for (const auto& column : request.predicate_columns) { - collect_scan_column(column); - } - for (const auto& column : request.non_predicate_columns) { - if (!request.is_count_star_placeholder(column.column_id())) { - collect_scan_column(column); - } - } -} - Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { if (!column_schema.type_descriptor.unsupported_reason.empty()) { @@ -155,6 +92,8 @@ Status validate_projected_leaves_supported(const ParquetColumnSchema& column_sch Status validate_requested_columns_supported( const std::vector>& file_schema, const format::FileScanRequest& request) { + // Validate the projected native-schema leaves before pruning: checking a physical carrier at + // a later read site can let an unsupported logical type silently pass when every row is pruned. auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { const auto local_id = projection.local_id(); if (local_id == format::ROW_POSITION_COLUMN_ID || @@ -176,33 +115,6 @@ Status validate_requested_columns_supported( return Status::OK(); } -std::vector build_page_cache_ranges( - const ::parquet::FileMetaData& metadata, - const std::vector>& file_schema, - const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { - std::unordered_set leaf_column_ids; - collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); - std::vector ranges; - ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); - for (const auto& row_group_plan_item : row_group_plan.row_groups) { - auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_chunk_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - DORIS_CHECK(size >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); - } - } - } - return ranges; -} - const ParquetColumnSchema& projected_root_schema( const std::vector>& file_schema, const format::LocalColumnIndex& projection) { @@ -213,11 +125,10 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const ParquetColumnReader& shape_reader, - int64_t expected_rows) { - const auto& def_levels = shape_reader.nested_definition_levels(); - const auto& rep_levels = shape_reader.nested_repetition_levels(); - const int64_t levels_written = shape_reader.nested_levels_written(); + const CountColumnReader& shape_reader, int64_t expected_rows) { + const auto& def_levels = shape_reader.definition_levels(); + const auto& rep_levels = shape_reader.repetition_levels(); + const int64_t levels_written = shape_reader.levels_written(); DORIS_CHECK(levels_written >= expected_rows); if (root_schema.max_repetition_level == 0) { DORIS_CHECK(levels_written == expected_rows); @@ -265,7 +176,7 @@ int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { const auto& type_descriptor = column_schema.type_descriptor; - return type_descriptor.physical_type == ::parquet::Type::INT96 || + return type_descriptor.physical_type == tparquet::Type::INT96 || (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); } @@ -346,10 +257,9 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem } static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.descriptor != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be // truncated bounds rather than values present in the file, so they are safe for pruning // but cannot be returned as exact aggregate results. @@ -386,14 +296,17 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _enable_mapping_varbinary(enable_mapping_varbinary) {} ParquetReader::~ParquetReader() = default; Status ParquetReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_parquet_profile.total_time); if (_io_ctx != nullptr && _io_ctx->should_stop) { return Status::EndOfFile("stop"); } @@ -414,6 +327,7 @@ Status ParquetReader::init(RuntimeState* state) { _state->enable_strict_mode = state->enable_strict_mode(); _state->scheduler.set_timezone(&state->timezone_obj()); _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + _state->scheduler.set_runtime_state(state); } int64_t merge_read_slice_size = -1; if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { @@ -421,16 +335,30 @@ Status ParquetReader::init(RuntimeState* state) { } _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); - // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + // Opening the file parses the footer before any row group can be scheduled. Keep this timer + // around the whole operation so footer/cache latency cannot disappear from a slow profile. + { + SCOPED_TIMER(_parquet_profile.parse_footer_time); + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); + } + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _state->file_context.native_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _state->file_context.native_footer_cache_hits); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); - if (_enable_mapping_timestamp_tz) { - for (auto& column_schema : _state->file_schema) { - apply_timestamp_tz_mapping(column_schema.get()); + { + SCOPED_TIMER(_parquet_profile.parse_meta_time); + RETURN_IF_ERROR(build_parquet_column_schema(_state->file_context.native_metadata->schema(), + &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } } } return Status::OK(); @@ -444,11 +372,12 @@ void ParquetReader::set_batch_size(size_t batch_size) { } Status ParquetReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_parquet_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } file_schema->clear(); - if (_state == nullptr || _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } @@ -471,8 +400,8 @@ std::unique_ptr ParquetReader::create_column_mapper( } Status ParquetReader::open(std::shared_ptr request) { - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } auto request_snapshot = request; @@ -525,19 +454,20 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.file_size = _file_description->file_size; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( - *_state->file_context.metadata, _state->file_context.file_reader.get(), - _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state)); + *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, + scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, + _state->runtime_state, &_state->file_context, + _parquet_profile.column_reader_profile())); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } - if (_state->enable_page_cache) { - _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, - *request_snapshot, row_group_plan)); - } + // Native page readers admit exact validated page payloads to cache. Do not pre-register whole + // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + if (_profile != nullptr) { + _state->scheduler.set_pruning_profile(&_parquet_profile); + } _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); @@ -546,8 +476,8 @@ Status ParquetReader::open(std::shared_ptr request) { } Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { - if (_state == nullptr || _state->file_context.file_reader == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } *rows = 0; @@ -646,9 +576,9 @@ int64_t ParquetReader::get_total_rows() const { Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } if (_should_stop()) { @@ -662,6 +592,14 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + // Aggregate pushdown bypasses the scheduler but still requires the exact pruned row-group set. + // Finish lazy remote probes here; normal scans keep them at current-row-group granularity. + RETURN_IF_ERROR(finalize_parquet_row_group_plans( + *_state->file_context.native_metadata, _state->file_schema, *_request, + _state->enable_bloom_filter, &_state->scan_plan, _state->timezone, + _state->runtime_state, &_state->file_context, _parquet_profile.column_reader_profile(), + _profile == nullptr ? nullptr : &_parquet_profile)); + for (const auto& aggregate_column : request.columns) { const auto local_id = aggregate_column.projection.local_id(); if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { @@ -677,10 +615,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - result->count += row_group_metadata->num_rows(); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + result->count += row_group_metadata.num_rows; } if (request.agg_type == TPushAggOp::type::COUNT) { if (request.columns.empty()) { @@ -703,31 +640,14 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - std::shared_ptr<::parquet::RowGroupReader> row_group; - try { - row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); - } catch (const ::parquet::ParquetException& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } catch (const std::exception& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } - - ParquetColumnReaderFactory column_reader_factory( - row_group, _state->file_context.schema->num_columns(), - &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), - _state->timezone, _state->enable_strict_mode, - _parquet_profile.scan_profile().column_reader_profile); - std::unique_ptr shape_reader; - RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( - root_schema, &count_projection, &shape_reader)); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(CountColumnReader::create( + _state->file_context.native_data_file(), _state->file_context.native_metadata, + row_group_plan.row_group_id, root_schema, &count_projection, + _state->file_context.native_io_ctx, + _state->file_context.native_page_cache_enabled, + _state->file_context.native_page_cache_file_key, + _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); int64_t row_group_cursor = 0; @@ -741,18 +661,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r while (range_rows_read < selected_range.length) { const int64_t batch_rows = std::min(_batch_size, selected_range.length - range_rows_read); - // COUNT(col) only needs the top-level NULL state. The shape reader loads - // def/rep levels from one representative leaf and does not build value_indices - // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string - // leaf, but the levels-only protocol still avoids Doris-side string - // materialization for that leaf. + int64_t rows_read = 0; RETURN_IF_ERROR(_stop_status_if_requested( - shape_reader->load_nested_levels_batch(batch_rows))); - _record_scan_rows(batch_rows); + shape_reader->read_levels(batch_rows, &rows_read))); + if (rows_read != batch_rows) { + return Status::Corruption( + "Parquet COUNT reader returned {} rows, expected {}", rows_read, + batch_rows); + } + _record_scan_rows(rows_read); result->count += - count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); - range_rows_read += batch_rows; - row_group_cursor += batch_rows; + count_loaded_non_null_values(root_schema, *shape_reader, rows_read); + range_rows_read += rows_read; + row_group_cursor += rows_read; } } } @@ -779,13 +700,30 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r auto& aggregate_column = result->columns[request_column_idx]; aggregate_column.projection = request.columns[request_column_idx].projection; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); - DORIS_CHECK(column_chunk != nullptr); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + DORIS_CHECK(leaf_schema->leaf_column_id >= 0 && + leaf_schema->leaf_column_id < + static_cast(row_group_metadata.columns.size())); + const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; + DORIS_CHECK(column_chunk.__isset.meta_data); + const auto& column_metadata = column_chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics && + detail::can_use_native_footer_min_max( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id))) { + safe_statistics = detail::sanitize_native_footer_statistics( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id)); + } const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, column_chunk->statistics(), _state->timezone); + *leaf_schema, safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", leaf_schema->name); @@ -807,7 +745,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } Status ParquetReader::close() { + SCOPED_TIMER(_parquet_profile.total_time); if (_state != nullptr) { + _state->scheduler.close(); _sync_page_cache_profile(); RETURN_IF_ERROR(_state->file_context.close()); } diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..da6135b81ae838 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,7 +46,7 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +89,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index eafb741a53161d..47e7c4c3a68af2 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -15,11 +15,11 @@ #include "format_v2/parquet/parquet_scan.h" -#include - #include +#include #include #include +#include #include #include #include @@ -35,37 +35,93 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "util/defer_op.h" +#include "util/time.h" namespace doris::format::parquet { -namespace { +namespace detail { + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats) { + if (std::ranges::any_of(positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return positions; + } + auto ordered = positions; + std::stable_sort(ordered.begin(), ordered.end(), [&](size_t left, size_t right) { + const auto score = [&](size_t position) { + const auto& sample = stats.at(position); + return sample.cost_per_input_row_ns / std::max(1.0 - sample.survival_ratio, 0.01); + }; + return score(left) < score(right); + }); + return ordered; +} -int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability) { + if (std::ranges::any_of(ordered_positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return ordered_positions; + } + std::vector result; + double reach_probability = 1; + for (const size_t position : ordered_positions) { + if (!result.empty() && reach_probability < minimum_reach_probability) { + break; + } + result.push_back(position); + reach_probability *= stats.at(position).survival_ratio; + } + return result; } -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { + constexpr size_t WARMUP_SAMPLES = 8; + constexpr size_t STEADY_STATE_INTERVAL = 16; + return samples < WARMUP_SAMPLES || batch_sequence % STEADY_STATE_INTERVAL == 0; } -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +} // namespace detail + +namespace { + +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request); + +bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; } -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; +bool is_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; } -bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { +bool is_data_page_type(tparquet::PageType::type page_type) { + return page_type == tparquet::PageType::DATA_PAGE || + page_type == tparquet::PageType::DATA_PAGE_V2; +} + +bool is_fully_dictionary_encoded_chunk(const tparquet::ColumnMetaData& column_metadata) { + if (!column_metadata.__isset.dictionary_page_offset || + column_metadata.dictionary_page_offset < 0) { return false; } - const auto& encoding_stats = column_metadata.encoding_stats(); + const auto& encoding_stats = column_metadata.encoding_stats; if (!encoding_stats.empty()) { bool has_dictionary_data_page = false; for (const auto& encoding_stat : encoding_stats) { @@ -81,7 +137,7 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { + for (const auto encoding : column_metadata.encodings) { if (is_dictionary_data_encoding(encoding)) { has_dictionary_encoding = true; continue; @@ -94,19 +150,39 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr || + const tparquet::ColumnMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || column_schema.type == nullptr || column_schema.max_repetition_level > 0) { return false; } - if (!column_schema.type_descriptor.is_string_like || - column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + bool is_supported_physical_type = false; + switch (column_metadata.type) { + case tparquet::Type::BYTE_ARRAY: + is_supported_physical_type = column_schema.type_descriptor.is_string_like; + break; + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + is_supported_physical_type = true; + break; + case tparquet::Type::BOOLEAN: + // Parquet booleans are PLAIN encoded and cannot have a dictionary page. + break; + } + if (!is_supported_physical_type) { + return false; + } + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_VARBINARY) { + // A table STRING predicate can be rewritten through a raw VARBINARY file slot. Evaluating + // it on dictionary Fields before the mapping expression is neither type-safe nor exact. return false; } - // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact - // only when every data page is dictionary encoded. Mixed dictionary/plain chunks are left on - // the normal decoded-value path, matching the safety rule used by StarRocks and Doris v1. + // The row filter consumes dictionary ids rather than decoded values, so a plain data page + // cannot resume this reader without changing its output domain. Keep mixed chunks on the + // normal decoded-value path to preserve one representation for the complete column chunk. return is_fully_dictionary_encoded_chunk(column_metadata); } @@ -143,35 +219,6 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, } } -bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, int row_group_idx) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start_offset = scan_range.start_offset; - const int64_t range_end_offset = range_start_offset + scan_range.size; - DORIS_CHECK(range_start_offset >= 0); - DORIS_CHECK(range_end_offset >= range_start_offset); - if (range_start_offset == 0 && - (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { - return false; - } - - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - DORIS_CHECK(row_group_metadata->num_columns() > 0); - const auto first_column = row_group_metadata->ColumnChunk(0); - const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); - DORIS_CHECK(first_column != nullptr); - DORIS_CHECK(last_column != nullptr); - const int64_t row_group_start_offset = column_start_offset(*first_column); - const int64_t row_group_end_offset = - column_start_offset(*last_column) + last_column->total_compressed_size(); - const int64_t row_group_mid_offset = - row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; - return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; -} - std::vector request_scan_columns(const format::FileScanRequest& request) { std::vector scan_columns; scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); @@ -212,10 +259,17 @@ void materialize_count_star_placeholders(const format::FileScanRequest& request, } } -std::vector build_row_group_prefetch_ranges( - const ::parquet::FileMetaData& metadata, +} // namespace + +namespace detail { + +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, - const std::vector& scan_columns, int row_group_idx) { + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); std::unordered_set leaf_column_ids; for (const auto& projection : scan_columns) { const auto local_id = projection.local_id(); @@ -223,92 +277,233 @@ std::vector build_row_group_prefetch_ranges( local_id == format::GLOBAL_ROWID_COLUMN_ID) { continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return Status::Corruption("Invalid Parquet projected column id {}", local_id); + } // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. // Example: for a struct column s, projecting only s.a should include only // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group index {}", row_group_idx); + } + const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); - std::vector ranges; - ranges.reserve(ordered_leaf_column_ids.size()); + ranges->reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(row_group_metadata.columns.size())) { + return Status::Corruption("Invalid Parquet leaf column id {}", leaf_column_id); + } + const auto& chunk = row_group_metadata.columns[leaf_column_id]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet leaf column {} has no chunk metadata", + leaf_column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range(chunk.meta_data, file_size, + parquet_816_padding, &chunk_range)); + if (chunk_range.length > 0) { + if (chunk_range.offset > static_cast(std::numeric_limits::max()) || + chunk_range.length > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet column chunk range exceeds int64 coordinates"); + } + // Prefetch must use the same checked chunk extent as the decoder, including the + // PARQUET-816 compatibility padding, so warm-up cannot target different bytes. + ranges->push_back( + ParquetPageCacheRange {.offset = static_cast(chunk_range.offset), + .size = static_cast(chunk_range.length)}); } } - return ranges; + return Status::OK(); } -Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, - std::vector* row_group_first_rows, - std::vector* selected_row_groups) { - DORIS_CHECK(row_group_first_rows != nullptr); - DORIS_CHECK(selected_row_groups != nullptr); - row_group_first_rows->assign(metadata.num_row_groups(), 0); +} // namespace detail + +namespace detail { + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + if (scan_range.start_offset < 0 || scan_range.size < -1 || + (scan_range.size >= 0 && + scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { + return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, + scan_range.size); + } + const uint64_t range_start = static_cast(scan_range.start_offset); + const uint64_t range_end = scan_range.size < 0 + ? std::numeric_limits::max() + : range_start + static_cast(scan_range.size); + const size_t file_size = scan_range.file_size < 0 ? std::numeric_limits::max() + : static_cast(scan_range.file_size); + const bool full_file_range = + scan_range.size < 0 || (range_start == 0 && scan_range.file_size >= 0 && + range_end >= static_cast(scan_range.file_size)); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + row_group_first_rows->assign(metadata.row_groups.size(), 0); selected_row_groups->clear(); - selected_row_groups->reserve(metadata.num_row_groups()); - int64_t next_row_group_first_row = 0; - for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows < 0) { + selected_row_groups->reserve(metadata.row_groups.size()); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { return Status::Corruption("Invalid negative row count in parquet row group {}", row_group_idx); } - next_row_group_first_row += row_group_rows; - if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { - selected_row_groups->push_back(row_group_idx); + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + next_first_row += row_group.num_rows; + bool selected = full_file_range; + if (!full_file_range) { + if (row_group.columns.empty()) { + return Status::Corruption("Parquet row group {} has no column chunks", + row_group_idx); + } + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + // Checked chunk ranges make end >= start; this midpoint form cannot overflow even + // when footer offsets are close to the host coordinate limit. + const uint64_t group_mid = + static_cast(group_start) + (group_end - group_start) / 2; + selected = group_mid >= range_start && group_mid < range_end; + } + if (selected) { + selected_row_groups->push_back(cast_set(row_group_idx)); + } + } + return Status::OK(); +} + +} // namespace detail + +namespace { + +std::vector intersect_row_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_end = left[left_idx].start + left[left_idx].length; + const int64_t right_end = right[right_idx].start + right[right_idx].length; + const int64_t start = std::max(left[left_idx].start, right[right_idx].start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back({.start = start, .length = end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +Status finalize_native_row_group_read_plan( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, + RowGroupReadPlan* row_group_plan, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + bool* selected) { + DORIS_CHECK(row_group_plan != nullptr && pruning_stats != nullptr && file_context != nullptr && + selected != nullptr); + *selected = true; + if (!row_group_plan->expensive_pruning_pending) { + return Status::OK(); + } + row_group_plan->expensive_pruning_pending = false; + const auto& thrift = metadata.to_thrift(); + const std::vector candidate {row_group_plan->row_group_id}; + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + thrift, file_schema, request, &candidate, &metadata_selected, enable_bloom_filter, + pruning_stats, timezone, runtime_state, file_context, column_reader_profile, + ParquetMetadataProbeMode::EXPENSIVE_ONLY)); + if (metadata_selected.empty()) { + *selected = false; + return Status::OK(); + } + + std::unordered_set requested_leaf_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); + } + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_plan->row_group_id, requested_leaf_ids, &page_indexes, + &pruning_stats->read_page_index_time, &pruning_stats->parse_page_index_time)); + } + std::vector page_selected_ranges; + std::map page_skip_plans; + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + thrift, page_indexes, file_schema, request, row_group_plan->row_group_rows, + &page_selected_ranges, &page_skip_plans, pruning_stats, timezone, runtime_state)); + row_group_plan->selected_ranges = + intersect_row_ranges(row_group_plan->selected_ranges, page_selected_ranges); + row_group_plan->page_skip_plans = std::move(page_skip_plans); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan->offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan->selected_ranges.empty()) { + *selected = false; + return Status::OK(); } + pruning_stats->selected_row_ranges += row_group_plan->selected_ranges.size(); return Status::OK(); } -Status build_row_group_read_plans( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_row_groups, const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); plan->row_groups.reserve(selected_row_groups.size()); - for (const auto row_group_idx : selected_row_groups) { - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows == 0) { + for (const int row_group_idx : selected_row_groups) { + const auto& row_group = thrift.row_groups[row_group_idx]; + if (row_group.num_rows == 0) { continue; } - RowGroupReadPlan row_group_plan; row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; - row_group_plan.row_group_rows = row_group_rows; - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + row_group_plan.row_group_rows = row_group.num_rows; + row_group_plan.selected_ranges = {{.start = 0, .length = row_group.num_rows}}; + row_group_plan.expensive_pruning_pending = true; plan->row_groups.push_back(std::move(row_group_plan)); } return Status::OK(); @@ -316,52 +511,58 @@ Status build_row_group_read_plans( } // namespace -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); - plan->pruning_stats = ParquetPruningStats {}; - - // Row-group planning flow: - // - // parquet footer row groups - // | - // v - // split byte-range candidates - // | - // v - // row-group metadata pruning - // statistics/ZoneMap -> dictionary -> bloom filter - // | - // v - // page-index pruning per selected row group - // | - // v - // RowGroupReadPlan with selected row ranges - // - // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs - // only for remaining row groups and produces selected row ranges; the scan scheduler later skips - // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for - // correctness. + plan->pruning_stats = {}; + plan->enable_bloom_filter = enable_bloom_filter; std::vector row_group_first_rows; - std::vector scan_range_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, - &scan_range_selected_row_groups)); - - std::vector metadata_selected_row_groups; + std::vector scan_range_selected; + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( + metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( - metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, - &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, - runtime_state)); + metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, + column_reader_profile, ParquetMetadataProbeMode::FOOTER_ONLY)); + RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, + metadata_selected, row_group_first_rows, plan, + timezone, runtime_state, file_context)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} - RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, - metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state)); +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + std::vector selected_plans; + selected_plans.reserve(plan->row_groups.size()); + for (auto& row_group_plan : plan->row_groups) { + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + metadata, file_schema, request, enable_bloom_filter, &row_group_plan, + &deferred_stats, timezone, runtime_state, file_context, column_reader_profile, + &selected)); + if (parquet_profile != nullptr) { + parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (selected) { + selected_plans.push_back(std::move(row_group_plan)); + } + } + plan->row_groups = std::move(selected_plans); plan->pruning_stats.selected_row_groups = plan->row_groups.size(); return Status::OK(); } @@ -623,6 +824,7 @@ std::vector filter_ranges_by_condition_cache(const std::vectorflush_profile(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + reader->flush_profile(); + } +} + +bool ParquetScanScheduler::finish_current_reader_batch_profiles() { + bool crossed_page = false; + // A scheduler batch is counted once even when several projected leaves cross page boundaries. + for (const auto& reader : _current_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + return crossed_page; +} + +const detail::PredicateConjunctSchedule& ParquetScanScheduler::predicate_conjunct_schedule( + const format::FileScanRequest& request) { + if (_predicate_schedule_request == &request) { + return _predicate_schedule; + } + + // FileScanRequest is frozen by ParquetReader::open(). Its address therefore identifies both + // the conjunct set and local-position mapping for the scheduler lifetime. + _predicate_schedule = build_predicate_conjunct_schedule(request); + _predicate_schedule_request = &request; + _predicate_positions_scratch.clear(); + _predicate_indices_by_position_scratch.clear(); + _predicate_positions_scratch.reserve(request.predicate_columns.size()); + _predicate_indices_by_position_scratch.reserve(request.predicate_columns.size()); + for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto position_it = + request.local_positions.find(request.predicate_columns[idx].column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + _predicate_positions_scratch.push_back(position); + _predicate_indices_by_position_scratch.emplace(position, idx); + } + return _predicate_schedule; +} + +std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const { + std::vector positions; + std::unordered_map columns_by_position; + positions.reserve(request.predicate_columns.size()); + columns_by_position.reserve(request.predicate_columns.size()); + for (const auto& column : request.predicate_columns) { + const auto position_it = request.local_positions.find(column.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + columns_by_position.emplace(position, &column); + } + auto ordered = detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + ordered = detail::adaptive_prefetch_prefix(ordered, _predicate_runtime_stats, 0.25); + std::vector result; + result.reserve(ordered.size()); + for (const size_t position : ordered) { + result.push_back(*columns_by_position.at(position)); + } + return result; +} + Status ParquetScanScheduler::open_next_row_group( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; - if (_next_row_group_plan_idx >= _row_group_plans.size()) { + RowGroupReadPlan* selected_plan = nullptr; + while (_next_row_group_plan_idx < _row_group_plans.size()) { + RowGroupReadPlan& candidate_plan = _row_group_plans[_next_row_group_plan_idx++]; + // Probe only the row group about to execute. This keeps LIMIT/cancellation latency + // independent of the number of later remote row groups while preserving eager footer + // statistics pruning during open. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + *file_context.native_metadata, file_schema, request, _enable_bloom_filter, + &candidate_plan, &deferred_stats, _timezone, _runtime_state, &file_context, + _scan_profile.column_reader_profile, &selected)); + if (_parquet_profile != nullptr) { + _parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (!selected) { + continue; + } + selected_plan = &candidate_plan; + break; + } + if (selected_plan == nullptr) { + // The last row group's native readers have already been released by + // reset_current_row_group(). Flush the shared merge reader now so its counters are visible + // when EOF is returned and its bounded scratch does not survive until file close. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; return Status::OK(); } - const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = *selected_plan; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. - // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be - // consumed as one forward pass, while the later RecordReader opens the same column chunk again - // for the data-page stream. + // Dictionary probes and data-page readers share the native metadata tree. Reset the previous + // row-group merge reader before probing because dictionary-page offsets are not scan ordered. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - try { - _current_row_group = file_context.file_reader->RowGroup(row_group_idx); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } - - auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - _current_row_group_rows = row_group_metadata->num_rows(); + + const auto& row_group_metadata = + file_context.native_metadata->to_thrift().row_groups[row_group_idx]; + _current_row_group_rows = row_group_metadata.num_rows; DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; + _has_current_row_group = true; DORIS_CHECK(!row_group_plan.selected_ranges.empty()); _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; + _current_offset_indexes = std::move(row_group_plan.offset_indexes); + // Condition Cache and split planning can narrow logical ranges without a physical OffsetIndex. + // Native readers must keep the sequential level/value cursor path valid in that case; only a + // PageIndex-derived skip plan requires the transferred indexes below. + for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { + if (!_current_offset_indexes.contains(leaf_column_id)) { + continue; + } + for (size_t page = 0; page < skip_plan.skipped_pages.size(); ++page) { + if (!skip_plan.should_skip_page(page)) { + continue; + } + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + skip_plan.skipped_page_compressed_size(page)); + } + } + } _current_range_idx = 0; _current_range_rows_read = 0; _current_predicate_columns.clear(); _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, - row_group_idx, *row_group_metadata)); - _current_merge_range_active = - prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); - - ParquetColumnReaderFactory column_reader_factory( - _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, - _page_skip_profile, _timezone, _enable_strict_mode, - _scan_profile.column_reader_profile); + row_group_idx, row_group_metadata)); + // Dictionary probing is complete, so the native data-page readers can now share the same + // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a + // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy + // materialization for wide schemas. + const auto& thrift_metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : std::string {}); + std::vector native_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); + _current_merge_range_active = file_context.set_native_random_access_ranges( + native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, + _merge_read_slice_size); + for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (_current_predicate_columns.contains(local_id)) { continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); + continue; + } + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR( - column_reader_factory.create(*column_schema, &col, &column_reader, - _current_dictionary_filters.contains(local_id))); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, + _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, + &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); } - // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still - // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks - // in the background and never changes the row/column materialization order. + // Start warming filter-column chunks as soon as their row group is selected. The native + // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never + // changes row/column materialization order. if (!_current_merge_range_active) { - prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, - &_current_predicate_prefetched); + const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); + RETURN_IF_ERROR(prefetch_current_row_group_columns( + file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { - const auto local_id = col.local_id(); + const auto local_id = col.column_id(); if (request.is_count_star_placeholder(col.column_id())) { continue; } - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_non_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_non_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_non_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_non_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, false, _scan_profile.column_reader_profile, + &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } - if (!_current_merge_range_active && request.conjuncts.empty() && - request.delete_conjuncts.empty()) { + if (!_current_merge_range_active && + ((request.conjuncts.empty() && request.delete_conjuncts.empty()) || + _predicate_survival_ratio >= 0.8)) { // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } *has_row_group = true; return Status::OK(); @@ -849,12 +1199,7 @@ Status ParquetScanScheduler::flush_pending_non_predicate_skip_rows() { namespace { -struct PredicateConjunctSchedule { - std::map single_column_conjuncts; - VExprContextSPtrs remaining_conjuncts; -}; - -PredicateConjunctSchedule build_predicate_conjunct_schedule( +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request) { std::unordered_set predicate_block_positions; predicate_block_positions.reserve(request.predicate_columns.size()); @@ -864,7 +1209,7 @@ PredicateConjunctSchedule build_predicate_conjunct_schedule( predicate_block_positions.insert(position_it->second.value()); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; for (const auto& conjunct : request.conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); @@ -962,21 +1307,11 @@ uint16_t count_selected_rows(const IColumn::Filter& filter) { return selected_rows; } -Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, - const IColumn::Filter& compact_filter) { - if (positions.empty()) { - return Status::OK(); - } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); - return Status::OK(); -} - IColumn::Filter build_dictionary_entry_filter(size_t block_position, const ParquetColumnSchema& column_schema, const VExprContextSPtrs& conjuncts, - const ParquetDictionaryWords& dict_words) { - auto fields = dictionary_fields_from_words(dict_words); - IColumn::Filter dictionary_filter(fields.size(), 1); + const IColumn& dictionary) { + IColumn::Filter dictionary_filter(dictionary.size(), 1); DictionaryEvalContext ctx; auto& slot = ctx.slots .emplace(static_cast(block_position), @@ -984,14 +1319,15 @@ IColumn::Filter build_dictionary_entry_filter(size_t block_position, .data_type = column_schema.type, .values = {}}) .first->second; slot.values.reserve(1); - - for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + for (size_t dictionary_id = 0; dictionary_id < dictionary.size(); ++dictionary_id) { + Field value; + dictionary.get(dictionary_id, value); slot.values.clear(); - slot.values.push_back(fields[dict_idx]); - dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch - ? 0 - : 1; + slot.values.push_back(std::move(value)); + dictionary_filter[dictionary_id] = VExprContext::evaluate_dictionary_filter( + conjuncts, ctx) == ZoneMapFilterResult::kNoMatch + ? 0 + : 1; } return dictionary_filter; } @@ -1002,16 +1338,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata) { + const tparquet::RowGroup& row_group_metadata) { _current_dictionary_filters.clear(); _current_dictionary_residual_conjuncts.clear(); if (request.conjuncts.empty()) { return Status::OK(); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; { SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); - schedule = build_predicate_conjunct_schedule(request); + schedule = predicate_conjunct_schedule(request); } if (schedule.single_column_conjuncts.empty()) { return Status::OK(); @@ -1019,8 +1355,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + const auto local_id = col.column_id(); + if (!local_id.is_valid() || local_id.value() >= static_cast(file_schema.size())) { continue; } const auto position_it = request.local_positions.find(col.column_id()); @@ -1036,29 +1372,38 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // This optimization is deliberately limited to single-column predicates with a dictionary // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a // dict-id prefilter and residual children keep the normal row-level expression path. - const auto& column_schema = file_schema[local_id]; + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || - column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + const auto& column_chunk = row_group_metadata.columns[column_schema->leaf_column_id]; + if (!column_chunk.__isset.meta_data || + !supports_row_level_dictionary_filter(*column_schema, column_chunk.meta_data)) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - ParquetDictionaryWords dict_words; + std::unique_ptr column_reader; + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_file, file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, true, _scan_profile.column_reader_profile, + &column_reader)); + MutableColumnPtr dictionary_values; { SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); - if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, - column_schema->leaf_column_id, *column_schema, - &dict_words)) { + auto dictionary_result = column_reader->dictionary_values(); + if (!dictionary_result.has_value()) { update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + // Dictionary filtering is optional: a probe failure must not reject a file that + // the normal native read path can still decode. continue; } + dictionary_values = std::move(dictionary_result).value(); } // Build a safe dictionary prefilter from the dictionary-filter interface instead of @@ -1069,15 +1414,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( DictionaryResidualConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, dict_words); + dictionary_filter = build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values); residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } // The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the - // predicate with an integer lookup and only materialize STRING values for surviving rows. + // predicate with an integer lookup and materialize typed values only for surviving rows. _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + _current_predicate_columns.emplace(local_id, std::move(column_reader)); update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); } return Status::OK(); @@ -1094,16 +1440,122 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { selection->resize(static_cast(batch_rows)); } - const auto schedule = build_predicate_conjunct_schedule(request); + const auto& schedule = predicate_conjunct_schedule(request); + std::unordered_set residual_predicate_positions; + auto remember_residual_positions = [&](const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + std::set positions; + conjunct->root()->collect_slot_column_ids(positions); + for (const int position : positions) { + if (position >= 0) { + residual_predicate_positions.insert(cast_set(position)); + } + } + } + }; + remember_residual_positions(schedule.remaining_conjuncts); + remember_residual_positions(request.delete_conjuncts); + const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); - std::vector read_column_positions; + auto& read_column_positions = _read_column_positions_scratch; + read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); + for (auto& rows : _predicate_column_selection_scratch | std::views::values) { + rows.clear(); + } + + auto remember_column_selection = [&](uint32_t position) { + auto& rows = _predicate_column_selection_scratch[position]; + rows.resize(*selected_rows); + for (uint16_t row = 0; row < *selected_rows; ++row) { + // SelectionVector and the scanner batch contract both bound row ordinals to uint16_t; + // keep the checked conversion explicit when persisting the coordinate mapping. + rows[row] = cast_set(selection->get_index(row)); + } + }; - auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, - ColumnId local_id, bool* used_dictionary_filter) -> Status { + auto compact_predicate_columns = [&](bool discard_predicate_only_payload) -> Status { + bool compacted = false; + int64_t compacted_bytes = 0; + for (const uint32_t position : read_column_positions) { + auto& source_rows = _predicate_column_selection_scratch[position]; + const auto& old_column = file_block->get_by_position(position).column; + if (old_column->size() != source_rows.size()) { + return Status::Corruption( + "Predicate column {} has {} values but {} remembered source rows", position, + old_column->size(), source_rows.size()); + } + bool predicate_only = false; + if (discard_predicate_only_payload) { + predicate_only = std::ranges::any_of( + request.predicate_only_columns, [&](format::LocalColumnId local_id) { + const auto position_it = request.local_positions.find(local_id); + return position_it != request.local_positions.end() && + position_it->second.value() == position; + }); + } + if (predicate_only) { + auto placeholder = old_column->clone_empty(); + // Hidden predicate values are dead after the last filter, but every file-block + // column must retain the selected row count until TableReader drops hidden slots. + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(position, std::move(placeholder)); + remember_column_selection(position); + continue; + } + bool already_compact = source_rows.size() == *selected_rows && + old_column->size() == static_cast(*selected_rows); + for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { + already_compact = source_rows[row] == selection->get_index(row); + } + if (already_compact) { + continue; + } + auto& filter = _predicate_compaction_filter_scratch; + // resize_fill() preserves bytes when the next predicate column is smaller. Clear the + // whole reusable mask so survivors from an earlier coordinate space cannot reappear. + filter.resize(source_rows.size()); + std::ranges::fill(filter, 0); + size_t source_idx = 0; + uint16_t selected_idx = 0; + while (source_idx < source_rows.size() && selected_idx < *selected_rows) { + const auto source_row = source_rows[source_idx]; + const auto selected_row = selection->get_index(selected_idx); + if (source_row < selected_row) { + ++source_idx; + continue; + } + DORIS_CHECK_EQ(source_row, selected_row); + filter[source_idx++] = 1; + ++selected_idx; + } + DORIS_CHECK_EQ(selected_idx, *selected_rows); + compacted_bytes += static_cast(old_column->byte_size()); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + position, old_column->filter(filter, *selected_rows))); + remember_column_selection(position); + compacted = true; + } + if (compacted) { + update_counter_if_not_null(_scan_profile.predicate_compaction_bytes, compacted_bytes); + update_counter_if_not_null(_scan_profile.predicate_compaction_count, 1); + } + // The output path must not apply a batch-coordinate filter to columns that now use compact + // coordinates. The loop above establishes this invariant even when no bytes moved because + // every column was already aligned. + *predicate_columns_filtered = !read_column_positions.empty(); + return Status::OK(); + }; + + auto read_predicate_column = + [&](ParquetColumnReader* column_reader, size_t block_position, + format::LocalColumnId local_id, const VExprContextSPtrs* single_column_conjuncts, + bool* used_dictionary_filter, bool* used_fixed_width_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); + DORIS_CHECK(used_fixed_width_filter != nullptr); *used_dictionary_filter = false; + *used_fixed_width_filter = false; DCHECK(remove_nullable(column_reader->type()) ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) << column_reader->type()->get_name() << " " @@ -1130,22 +1582,72 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, filtered_rows); if (new_selected_rows != selected_rows_before) { - // The dictionary reader has already appended only surviving values for the - // current column. Apply the compact row filter only to columns read before this - // one, then update the shared selection for later predicate/lazy columns. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); + // The dictionary reader already appended only survivors for this column. Keep + // older predicate columns in their original coordinate spaces and compact all + // of them once at the expression/output boundary below. *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); *used_dictionary_filter = true; return Status::OK(); } } + if (single_column_conjuncts != nullptr && + !residual_predicate_positions.contains(block_position)) { + VExprSPtrs direct_conjuncts; + direct_conjuncts.reserve(single_column_conjuncts->size()); + std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), + [](const auto& context) { return context->root(); }); + if (!direct_conjuncts.empty()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + const bool predicate_only = request.is_predicate_only(local_id); + // The raw decoder cannot rewind after evaluating encoded fixed-width values. + // Project survivors in that pass when output still needs the predicate column. + IColumn* projected_column = predicate_only ? nullptr : column.get(); + RETURN_IF_ERROR(column_reader->select_with_fixed_width_filter( + *selection, *selected_rows, batch_rows, direct_conjuncts, + cast_set(block_position), projected_column, &compact_filter, + &used_filter)); + if (used_filter) { + DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_batches, + 1); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_rows, + selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + if (new_selected_rows != selected_rows_before) { + *selected_rows = apply_compact_filter_to_selection( + compact_filter, selection, selected_rows_before); + } + if (predicate_only) { + // This slot is absent from every residual/delete conjunct, so no later + // expression can observe its payload. Keep only the block row-shape contract. + auto placeholder = column->clone_empty(); + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } else { + file_block->replace_by_position(block_position, std::move(column)); + } + read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); + *predicate_columns_filtered = true; + *used_fixed_width_filter = true; + return Status::OK(); + } + } + } + if (*selected_rows == batch_rows) { int64_t column_rows = 0; RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); @@ -1166,6 +1668,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); return Status::OK(); }; @@ -1187,16 +1690,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // All columns read so far are already compacted to the current selection. Apply the - // compact filter to those columns and the selection vector together, so later predicate - // columns can read only rows that survived previous predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1220,16 +1717,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // Dictionary-covered children have already reduced the compact block. Apply only the - // residual child filters here, then keep the same compacted-column invariant as the - // normal conjunct path for later predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1266,24 +1757,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, compact_filter.resize_fill(selected_rows_before, 0); } if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; auto read_all_predicate_columns = [&]() -> Status { for (const auto& [fid, column_reader] : _current_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), - fid, &used_dictionary_filter)); + fid, nullptr, &used_dictionary_filter, + &used_fixed_width_filter)); } return Status::OK(); }; @@ -1303,45 +1793,69 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. - for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + _ordered_predicate_positions_scratch = detail::order_adaptive_predicates( + _predicate_positions_scratch, _predicate_runtime_stats); + const auto& ordered_positions = _ordered_predicate_positions_scratch; + for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { + const size_t position = ordered_positions[order_idx]; + const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; - const auto fid = col.local_id(); + const auto fid = col.column_id(); auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); + const uint16_t rows_before = *selected_rows; + auto& stats = _predicate_runtime_stats[position]; + const bool sample = detail::should_sample_adaptive_predicate(stats.samples, + predicate_batch_sequence); + const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + const VExprContextSPtrs* column_conjuncts = + conjunct_it == schedule.single_column_conjuncts.end() ? nullptr + : &conjunct_it->second; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, - &used_dictionary_filter)); - if (*selected_rows == 0) { - for (size_t remaining_idx = idx + 1; - remaining_idx < request.predicate_columns.size(); ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); - auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); - DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); - RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + column_conjuncts, &used_dictionary_filter, + &used_fixed_width_filter)); + if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else if (!used_fixed_width_filter) { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } - return Status::OK(); - } - const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); - if (conjunct_it == schedule.single_column_conjuncts.end()) { - continue; } - if (used_dictionary_filter) { - const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); - DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); - RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( - residual_it->second)); - } else { - RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + if (sample) { + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; + } else { + stats.cost_per_input_row_ns = + ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + } + ++stats.samples; } if (*selected_rows != 0) { continue; } - for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); - ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + for (size_t remaining_order_idx = order_idx + 1; + remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { + const size_t remaining_idx = _predicate_indices_by_position_scratch.at( + ordered_positions[remaining_order_idx]); + const auto remaining_fid = request.predicate_columns[remaining_idx].column_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); @@ -1351,47 +1865,59 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; + auto compact_predicate_columns_with_profile = + [&](bool discard_predicate_only_payload) -> Status { + const int64_t start_ns = MonotonicNanos(); + auto status = compact_predicate_columns(discard_predicate_only_payload); + update_counter_if_not_null(_scan_profile.predicate_compaction_time, + MonotonicNanos() - start_ns); + return status; + }; + RETURN_IF_ERROR(read_round_by_round()); + // Single-column expressions only touch the just-read column, so earlier columns can retain + // their own row mappings. Compact only when a later expression needs a shared coordinate + // space; otherwise the final boundary can discard hidden predicate payloads without scanning + // them again. + if (!schedule.remaining_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); - if (_scan_profile.predicate_filter_time == nullptr) { - return execute_scheduled_delete_conjuncts(); + if (!request.delete_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); } - SCOPED_TIMER(_scan_profile.predicate_filter_time); - return execute_scheduled_delete_conjuncts(); -} - -bool ParquetScanScheduler::prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx) { - if (file_context.metadata == nullptr) { - return false; + if (_scan_profile.predicate_filter_time == nullptr) { + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); + } else { + SCOPED_TIMER(_scan_profile.predicate_filter_time); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - const auto ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); - const size_t avg_io_size = detail::average_prefetch_range_size(ranges); - return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, - _merge_read_slice_size); + return compact_predicate_columns_with_profile(true); } -void ParquetScanScheduler::prefetch_current_row_group_columns( +Status ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || - _current_row_group_id < 0 || file_context.metadata == nullptr) { - return; + _current_row_group_id < 0 || file_context.native_metadata == nullptr) { + return Status::OK(); } *prefetched = true; // The scanner request separates predicate and non-predicate columns so Parquet can read // predicate columns first and lazily materialize the rest. Keep the same contract for // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. - file_context.prefetch_ranges( - build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, - _current_row_group_id), - nullptr); + const auto& metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + std::vector ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + metadata, file_schema, scan_columns, _current_row_group_id, + file_context.native_file->size(), compat.parquet_816_padding, &ranges)); + file_context.prefetch_ranges(ranges, nullptr); + return Status::OK(); } Status ParquetScanScheduler::read_current_row_group_batch( @@ -1399,6 +1925,24 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { + // Reader statistics are cumulative plain integers. Publishing their delta recursively for + // every tiny batch is measurable on wide/nested scans, so flush periodically and force the + // tail at row-group reset/close. + Defer profile_flush {[this, batch_rows]() { + // A widened predicate batch can be emitted in several output slices. Its lazy readers + // have not consumed the whole physical batch until the last slice is drained. + if (_pending_predicate_selection.empty() && finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && + _current_range_rows_read + batch_rows == + _current_selected_ranges[_current_range_idx].length; + if (++_batches_since_profile_flush >= PROFILE_FLUSH_BATCH_INTERVAL || finishes_row_group) { + flush_current_reader_profiles(); + _batches_since_profile_flush = 0; + } + }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } @@ -1414,7 +1958,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } return Status::OK(); } - SelectionVector selection; + auto& selection = _selection; DORIS_CHECK(batch_rows <= std::numeric_limits::max()); uint16_t selected_rows = static_cast(batch_rows); int64_t conjunct_filtered_rows = 0; @@ -1425,6 +1969,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); const bool need_filter_output = selected_rows != batch_rows; + const double batch_survival = static_cast(selected_rows) / batch_rows; + _predicate_survival_ratio = _predicate_survival_ratio < 0 + ? batch_survival + : 0.25 * batch_survival + 0.75 * _predicate_survival_ratio; if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); } @@ -1437,6 +1985,11 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } else if (static_cast(selected_rows) == batch_rows && + _scan_profile.dense_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.dense_batches, 1); + } else if (_scan_profile.selected_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_batches, 1); } if (need_filter_output && !predicate_columns_filtered) { IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); @@ -1451,7 +2004,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0) { // Predicate readers have consumed this physical batch, but touching every lazy column here - // turns a long rejected prefix into `empty_batches * lazy_columns` Arrow calls. Record only + // turns a long rejected prefix into `empty_batches * lazy_columns` native calls. Record only // the positional lag. If [0, 32), [32, 64), and [64, 96) are empty, the first surviving // batch performs one skip(96) per lazy column. If the row group ends instead, reset drops the // lazy readers without flushing because no value from them can be observed. @@ -1466,9 +2019,30 @@ Status ParquetScanScheduler::read_current_row_group_batch( // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); + } + + if (selected_rows > _batch_size) { + DORIS_CHECK(_pending_predicate_selection.empty()); + _pending_predicate_batch_rows = batch_rows; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.resize(selected_rows); + for (uint16_t idx = 0; idx < selected_rows; ++idx) { + _pending_predicate_selection[idx] = + static_cast(selection.get_index(idx)); + } + for (const auto& col : request.predicate_columns) { + const auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t block_position = position_it->second.value(); + const auto& column = file_block->get_by_position(block_position).column; + DORIS_CHECK_EQ(column->size(), selected_rows); + _pending_predicate_columns.emplace(block_position, column); + } + return materialize_pending_predicate_batch(request, file_block, rows); } { @@ -1477,7 +2051,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( // selection vector. This also merges pending range gaps with fully filtered batches. RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); for (const auto& [fid, column_reader] : _current_non_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); auto column = file_block->get_by_position(block_position).column->assert_mutable(); @@ -1513,6 +2087,76 @@ Status ParquetScanScheduler::read_current_row_group_batch( return Status::OK(); } +Status ParquetScanScheduler::materialize_pending_predicate_batch( + const format::FileScanRequest& request, Block* file_block, size_t* rows) { + DORIS_CHECK(!_pending_predicate_selection.empty()); + DORIS_CHECK(_pending_predicate_selected_offset < _pending_predicate_selection.size()); + const size_t remaining_selected = + _pending_predicate_selection.size() - _pending_predicate_selected_offset; + const size_t output_rows = + std::min(static_cast(_batch_size), remaining_selected); + const size_t output_end = _pending_predicate_selected_offset + output_rows; + const int64_t physical_end = + output_end == _pending_predicate_selection.size() + ? _pending_predicate_batch_rows + : static_cast(_pending_predicate_selection[output_end - 1]) + 1; + DORIS_CHECK(physical_end > _pending_predicate_batch_rows_consumed); + const int64_t physical_rows = physical_end - _pending_predicate_batch_rows_consumed; + + _pending_output_selection.resize(output_rows); + for (size_t idx = 0; idx < output_rows; ++idx) { + const int64_t physical_row = + _pending_predicate_selection[_pending_predicate_selected_offset + idx]; + DORIS_CHECK(physical_row >= _pending_predicate_batch_rows_consumed); + _pending_output_selection.set_index( + idx, static_cast(physical_row - + _pending_predicate_batch_rows_consumed)); + } + + for (const auto& [block_position, column] : _pending_predicate_columns) { + file_block->replace_by_position( + block_position, column->cut(_pending_predicate_selected_offset, output_rows)); + } + { + SCOPED_TIMER(_scan_profile.column_read_time); + RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); + for (const auto& [fid, column_reader] : _current_non_predicate_columns) { + auto position_it = request.local_positions.find(fid); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + auto column = file_block->get_by_position(block_position).column->assert_mutable(); + [[maybe_unused]] const auto old_size = column->size(); + RETURN_IF_ERROR(column_reader->select(_pending_output_selection, + static_cast(output_rows), physical_rows, + column)); + if (column->size() != old_size + output_rows) { + return Status::Corruption( + "Parquet pending output column {} returned {} rows, expected {} rows", + column_reader->name(), column->size(), old_size + output_rows); + } + file_block->replace_by_position(block_position, std::move(column)); + } + } + materialize_count_star_placeholders(request, output_rows, file_block); + *rows = output_rows; + _pending_predicate_batch_rows_consumed = physical_end; + _pending_predicate_selected_offset = output_end; + if (_pending_predicate_selected_offset == _pending_predicate_selection.size()) { + DORIS_CHECK_EQ(_pending_predicate_batch_rows_consumed, _pending_predicate_batch_rows); + if (finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + _pending_predicate_batch_rows = 0; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.clear(); + _pending_predicate_columns.clear(); + _pending_output_selection.clear(); + } + return Status::OK(); +} + void ParquetScanScheduler::mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row) { @@ -1536,8 +2180,28 @@ Status ParquetScanScheduler::read_next_batch( const std::vector>& file_schema, const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; + if (!_pending_predicate_selection.empty()) { + RETURN_IF_ERROR(materialize_pending_predicate_batch(request, file_block, rows)); + *eof = false; + return Status::OK(); + } + int64_t predicate_batch_rows = _batch_size; + const int64_t max_predicate_batch_rows = std::min( + std::numeric_limits::max(), + std::max(DEFAULT_READ_BATCH_SIZE, _runtime_state == nullptr + ? DEFAULT_READ_BATCH_SIZE + : _runtime_state->batch_size())); + auto grow_empty_predicate_batch = [max_predicate_batch_rows](int64_t current) { + for (const int64_t target : + {int64_t {256}, int64_t {1024}, int64_t {4096}, max_predicate_batch_rows}) { + if (current < target) { + return std::min(target, max_predicate_batch_rows); + } + } + return max_predicate_batch_rows; + }; while (true) { - if (_current_row_group == nullptr) { + if (!_has_current_row_group) { bool has_row_group = false; RETURN_IF_ERROR( open_next_row_group(file_context, file_schema, request, &has_row_group)); @@ -1572,7 +2236,7 @@ Status ParquetScanScheduler::read_next_batch( continue; } - const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t batch_rows = std::min(predicate_batch_rows, remaining_rows); const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; @@ -1585,6 +2249,10 @@ Status ParquetScanScheduler::read_next_batch( _current_range_rows_read = 0; } if (*rows == 0) { + // Fully rejected probes carry no output-width sample. Widen predicate work to cross + // long empty prefixes cheaply; a later non-empty probe is sliced before lazy columns + // are materialized, so this internal width cannot escape the caller's row cap. + predicate_batch_rows = grow_empty_predicate_batch(predicate_batch_rows); continue; } *eof = false; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index c656b146cefeaa..dc926c3a7666e7 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include @@ -33,13 +36,6 @@ #include "runtime/runtime_profile.h" #include "storage/segment/condition_cache.h" -namespace parquet { -class FileMetaData; -class ParquetFileReader; -class RowGroupMetaData; -class RowGroupReader; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -57,6 +53,40 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +struct ParquetPageCacheRange; +struct ParquetScanRange; +class NativeParquetMetadata; + +namespace detail { +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + +struct AdaptivePredicateStats { + double cost_per_input_row_ns = 0; + double survival_ratio = 1; + size_t samples = 0; +}; + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups); +} // namespace detail // ============================================================================ // ============================================================================ @@ -74,23 +104,39 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely + // Deferred planning transfers parsed indexes to execution so narrowed scans never issue the + // same remote index reads a second time while opening the row group. + std::unordered_map offset_indexes; + // Footer statistics are cheap and eager. Remote dictionary/Bloom/page-index probes fill the + // remaining fields only when this row group reaches the scheduler. + bool expensive_pruning_pending = false; }; struct RowGroupScanPlan { std::vector row_groups; // row groups selected after pruning ParquetPruningStats pruning_stats; // pruning statistics + bool enable_bloom_filter = false; }; // ============================================================================ // ============================================================================ -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); + +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile = nullptr); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -116,6 +162,9 @@ class ParquetScanScheduler { _page_skip_profile = page_skip_profile; } void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_pruning_profile(const ParquetProfile* parquet_profile) { + _parquet_profile = parquet_profile; + } void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { _profile = profile; _merge_read_slice_size = merge_read_slice_size; @@ -128,6 +177,10 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + // Release row-group readers before the owning RuntimeProfile is reported. Native readers + // publish their accumulated page/decode statistics from their destructor. + void close() { reset_current_row_group(); } // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when // splitting selected row ranges into physical read batches. void set_batch_size(size_t batch_size) { @@ -145,7 +198,15 @@ class ParquetScanScheduler { bool* eof); private: + static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; + void reset_current_row_group(); + void flush_current_reader_profiles(); + bool finish_current_reader_batch_profiles(); + const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( + const format::FileScanRequest& request); + std::vector adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const; Status open_next_row_group(ParquetFileContext& file_context, const std::vector>& file_schema, @@ -163,38 +224,40 @@ class ParquetScanScheduler { ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata); + const tparquet::RowGroup& row_group_metadata); - void prefetch_current_row_group_columns( + Status prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); - bool prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx); - Status read_current_row_group_batch( ParquetFileContext& file_context, const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows); + Status materialize_pending_predicate_batch(const format::FileScanRequest& request, + Block* file_block, size_t* rows); + void mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row); std::vector _row_group_plans; // row group queue to scan size_t _next_row_group_plan_idx = 0; // index of the next row group to process - std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader - std::map> + bool _has_current_row_group = false; + // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. + std::unordered_map _current_offset_indexes; + // File-local ids are signed because virtual columns use reserved negative values. Keeping the + // typed id as the map key prevents GLOBAL_ROWID_COLUMN_ID from wrapping to a storage ColumnId. + std::map> _current_predicate_columns; // predicate ColumnReaders - std::map> + std::map> _current_non_predicate_columns; // non-predicate ColumnReaders - std::map + std::map _current_dictionary_filters; // local id -> dict entry bitmap - std::map>> + std::map>> _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts int64_t _current_row_group_rows = 0; // current row group row count int _current_row_group_id = -1; // current row group id in parquet metadata @@ -208,18 +271,46 @@ class ParquetScanScheduler { // readers can lag behind across fully filtered batches and range gaps; the lag is flushed once // before the next surviving batch is materialized, or discarded with the row group. int64_t _pending_non_predicate_skip_rows = 0; + // Empty predicate batches may widen their physical probe. If the first non-empty probe finds + // more rows than the caller's cap, keep its narrow predicate result here and materialize lazy + // columns in capped physical slices on subsequent calls. + int64_t _pending_predicate_batch_rows = 0; + int64_t _pending_predicate_batch_rows_consumed = 0; + size_t _pending_predicate_selected_offset = 0; + std::vector _pending_predicate_selection; + std::map _pending_predicate_columns; + SelectionVector _pending_output_selection; bool _current_predicate_prefetched = false; bool _current_non_predicate_prefetched = false; bool _current_merge_range_active = false; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; + const ParquetProfile* _parquet_profile = nullptr; RuntimeProfile* _profile = nullptr; int64_t _merge_read_slice_size = -1; std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + bool _enable_bloom_filter = false; + RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without + // reallocating selection indices, dense filter bytes, or compacted-column positions. + SelectionVector _selection; + std::vector _read_column_positions_scratch; + const format::FileScanRequest* _predicate_schedule_request = nullptr; + detail::PredicateConjunctSchedule _predicate_schedule; + std::vector _predicate_positions_scratch; + std::unordered_map _predicate_indices_by_position_scratch; + std::vector _ordered_predicate_positions_scratch; + std::unordered_map> + _predicate_column_selection_scratch; + IColumn::Filter _predicate_compaction_filter_scratch; + size_t _predicate_batch_sequence = 0; + size_t _batches_since_profile_flush = 0; + std::unordered_map _predicate_runtime_stats; + double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; int64_t _predicate_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index ff72c6982de301..7d1afb39240ac1 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -15,15 +15,6 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -48,15 +39,99 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" +#include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/thrift_util.h" +#include "util/unaligned.h" namespace doris::format::parquet { +namespace detail { + +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size) { + if (offset < 0 || header_size == 0 || payload_size < segment_v2::BloomFilter::MINIMUM_BYTES || + payload_size > segment_v2::BloomFilter::MAXIMUM_BYTES || payload_size % 32 != 0) { + return Status::Corruption( + "Invalid Parquet Bloom filter layout: offset {}, header {}, payload {}", offset, + header_size, payload_size); + } + const uint64_t unsigned_offset = static_cast(offset); + const uint64_t total_size = static_cast(header_size) + payload_size; + if (unsigned_offset > file_size || total_size > file_size - unsigned_offset) { + return Status::Corruption("Parquet Bloom filter range exceeds file size {}", file_size); + } + if (declared_length >= 0) { + const uint64_t unsigned_declared_length = static_cast(declared_length); + if (unsigned_declared_length < total_size || + unsigned_declared_length > file_size - unsigned_offset) { + return Status::Corruption( + "Parquet Bloom filter requires {} bytes, metadata declares {}, file has {}", + total_size, declared_length, file_size - unsigned_offset); + } + } + return Status::OK(); +} + +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id) { + return leaf_column_id >= 0 && metadata.__isset.column_orders && + leaf_column_id < static_cast(metadata.column_orders.size()) && + metadata.column_orders[leaf_column_id].__isset.TYPE_ORDER; +} + +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + auto sanitized = statistics; + if (!has_type_defined_order || !sanitized.__isset.min_value || !sanitized.__isset.max_value) { + sanitized.__isset.min_value = false; + sanitized.__isset.max_value = false; + sanitized.min_value.clear(); + sanitized.max_value.clear(); + } + const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (!sanitized.__isset.min || !sanitized.__isset.max || + (binary && sanitized.min != sanitized.max)) { + sanitized.__isset.min = false; + sanitized.__isset.max = false; + sanitized.min.clear(); + sanitized.max.clear(); + } + return sanitized; +} + +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + // Inexact bounds remain useful for pruning, but returning them as aggregate values changes the + // query result. Missing exactness fields are legacy-compatible; only an explicit false rejects. + if ((statistics.__isset.is_min_value_exact && !statistics.is_min_value_exact) || + (statistics.__isset.is_max_value_exact && !statistics.is_max_value_exact)) { + return false; + } + const auto sanitized = + sanitize_native_footer_statistics(type_descriptor, statistics, has_type_defined_order); + return (sanitized.__isset.min_value && sanitized.__isset.max_value) || + (sanitized.__isset.min && sanitized.__isset.max); +} + +} // namespace detail + namespace { +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone); + enum class ParquetRowGroupPruneReason { NONE, // cannot prune; must read STATISTICS, // excluded by ZoneMap statistics @@ -64,6 +139,60 @@ enum class ParquetRowGroupPruneReason { BLOOM_FILTER, // excluded by bloom filter }; +Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, + const io::FileReaderSPtr& file, io::IOContext* io_ctx, + std::unique_ptr* result) { + if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { + return Status::NotSupported("Parquet Bloom filter is unavailable"); + } + constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; + if (metadata.bloom_filter_offset < 0 || + (metadata.__isset.bloom_filter_length && metadata.bloom_filter_length <= 0)) { + return Status::Corruption("Invalid Parquet Bloom filter offset or declared length"); + } + const uint64_t bloom_offset = static_cast(metadata.bloom_filter_offset); + if (bloom_offset >= file->size()) { + return Status::Corruption("Parquet Bloom filter offset exceeds file size {}", file->size()); + } + const size_t available = file->size() - bloom_offset; + const size_t declared_available = + metadata.__isset.bloom_filter_length + ? std::min(metadata.bloom_filter_length, available) + : available; + const size_t header_read_size = std::min(declared_available, MAX_BLOOM_HEADER_BYTES); + std::vector header_buffer(header_read_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, + Slice(header_buffer.data(), header_buffer.size()), &bytes_read, + io_ctx)); + tparquet::BloomFilterHeader header; + uint32_t header_size = cast_set(bytes_read); + RETURN_IF_ERROR(deserialize_thrift_msg(header_buffer.data(), &header_size, true, &header)); + if (!header.algorithm.__isset.BLOCK || !header.compression.__isset.UNCOMPRESSED || + !header.hash.__isset.XXHASH || header.numBytes <= 0) { + return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); + } + + // Validate the complete split-block layout before allocating or adding footer-controlled + // offsets; BloomFilter::init() otherwise receives a truncated or oversized backing buffer. + RETURN_IF_ERROR(detail::validate_native_bloom_filter_layout( + metadata.bloom_filter_offset, header_size, header.numBytes, + metadata.__isset.bloom_filter_length ? metadata.bloom_filter_length : -1, + file->size())); + + std::vector data(cast_set(header.numBytes)); + RETURN_IF_ERROR(file->read_at(static_cast(metadata.bloom_filter_offset) + header_size, + Slice(data.data(), data.size()), &bytes_read, io_ctx)); + if (bytes_read != data.size()) { + return Status::Corruption("Truncated Parquet Bloom filter payload"); + } + auto bloom_filter = std::make_unique(); + RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), + segment_v2::HashStrategyPB::XX_HASH_64)); + *result = std::move(bloom_filter); + return Status::OK(); +} + bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { if (column_schema.type == nullptr) { return false; @@ -109,8 +238,15 @@ Status read_decoded_field(const ParquetColumnSchema& column_schema, DecodedColum view.fixed_length = column_schema.type_descriptor.fixed_length; view.timestamp_is_adjusted_to_utc = column_schema.type_descriptor.timestamp_is_adjusted_to_utc; view.timezone = timezone; - return column_schema.type->get_serde()->read_field_from_decoded_value(*column_schema.type, - field, view); + // Statistics are pruning proofs, not row materialization. A malformed non-NULL bound must + // disable pruning instead of being converted to NULL under permissive scan semantics. + view.enable_strict_mode = true; + RETURN_IF_ERROR(column_schema.type->get_serde()->read_field_from_decoded_value( + *column_schema.type, field, view)); + if (field->is_null()) { + return Status::DataQualityError("Non-NULL Parquet statistic decoded as NULL"); + } + return Status::OK(); } template @@ -172,30 +308,6 @@ bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics return !(column_statistics.max_value < column_statistics.min_value); } -template -bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics>(statistics); - const auto& min_value = typed_statistics->min(); - const auto& max_value = typed_statistics->max(); - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value) || - !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, - timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); -} - bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, const StringRef& value, Field* field, const cctz::time_zone* timezone) { @@ -206,54 +318,6 @@ bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedV return read_decoded_field(column_schema, view, field, timezone).ok(); } -bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - switch (statistics->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( - statistics); - const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); - const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { - return false; - } - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( - statistics); - const int type_length = column_schema.descriptor->type_length(); - const std::string min(reinterpret_cast(typed_statistics->min().ptr), - type_length); - const std::string max(reinterpret_cast(typed_statistics->max().ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - default: - return false; - } -} - template T load_predicate_value(const char* data) { T value; @@ -306,61 +370,19 @@ std::optional convert_logical_integer_to_physical_int32( return physical_value; } -class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { public: - ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, - const ::parquet::BloomFilter& bloom_filter) + NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const segment_v2::BloomFilter& bloom_filter) : _column_schema(column_schema), _bloom_filter(bloom_filter) {} - void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } + void add_bytes(const char*, size_t) override { DORIS_CHECK(false); } bool test_bytes(const char* buf, size_t size) const override { - if (buf == nullptr) { - return true; - } - // Parquet bloom filters are populated from the physical column carrier, while VExpr - // literals are materialized as Doris logical values. Keep the logical type in - // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet - // physical representation only at this adapter boundary. - switch (_column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return test_boolean(buf, size); - case ::parquet::Type::INT32: - return test_physical_int32(buf, size); - case ::parquet::Type::INT64: - return test_int64(buf, size); - case ::parquet::Type::FLOAT: - return test_float(buf, size); - case ::parquet::Type::DOUBLE: - return test_double(buf, size); - case ::parquet::Type::BYTE_ARRAY: - return test_byte_array(buf, size); - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return test_fixed_len_byte_array(buf, size); - default: - return true; + if (buf == nullptr || + _column_schema.type_descriptor.physical_type != tparquet::Type::INT32) { + return _bloom_filter.test_bytes(buf, size); } - } - - void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } - bool has_null() const override { return false; } - void add_hash(uint64_t hash) override { DORIS_CHECK(false); } - bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } - -private: - bool test_boolean(const char* buf, size_t size) const { - if (size == sizeof(bool)) { - const int32_t value = load_predicate_value(buf) ? 1 : 0; - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - if (size == sizeof(int32_t)) { - const int32_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); - } - return true; - } - - bool test_physical_int32(const char* buf, size_t size) const { const auto logical_value = load_predicate_integral_value(buf, size); if (!logical_value.has_value()) { return true; @@ -370,57 +392,20 @@ class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { if (!physical_value.has_value()) { return false; } - return find_int32(*physical_value); - } - - bool test_int64(const char* buf, size_t size) const { - if (size != sizeof(int64_t)) { - return true; - } - const int64_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_float(const char* buf, size_t size) const { - if (size != sizeof(float)) { - return true; - } - const float value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_double(const char* buf, size_t size) const { - if (size != sizeof(double)) { - return true; - } - const double value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_byte_array(const char* buf, size_t size) const { - ::parquet::ByteArray value(static_cast(size), - reinterpret_cast(buf)); - return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); - } - - bool test_fixed_len_byte_array(const char* buf, size_t size) const { - if (_column_schema.type_descriptor.fixed_length <= 0) { - return true; - } - if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { - return false; - } - ::parquet::FLBA value(reinterpret_cast(buf)); - return _bloom_filter.FindHash( - _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); + // Native file Bloom bytes are hashed from the Parquet physical carrier, not the wider + // Doris logical literal used by VExpr (for example UINT32 is exposed as BIGINT). + return _bloom_filter.test_bytes(reinterpret_cast(&*physical_value), + sizeof(*physical_value)); } - bool find_int32(int32_t value) const { - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.test_hash(hash); } +private: const ParquetColumnSchema& _column_schema; - const ::parquet::BloomFilter& _bloom_filter; + const segment_v2::BloomFilter& _bloom_filter; }; bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { @@ -428,14 +413,14 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { return false; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BOOLEAN: + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::BYTE_ARRAY: return true; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return column_schema.type_descriptor.is_string_like && column_schema.type_descriptor.fixed_length > 0; default: @@ -443,220 +428,6 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { } } -bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - if (!bloom_filter_supported(column_schema)) { - return false; - } - ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema.type, - .bloom_filter = &adapter, - }); - return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; -} - -struct RowGroupBloomFilterCache { - using CacheKey = std::pair; - - ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; - std::map> column_bloom_filters; - std::set loaded_columns; - - ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_reader == nullptr || leaf_column_id < 0) { - return nullptr; - } - const CacheKey cache_key {row_group_idx, leaf_column_id}; - if (loaded_columns.find(cache_key) == loaded_columns.end()) { - loaded_columns.insert(cache_key); - try { - std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; - if (pruning_stats != nullptr) { - SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } else { - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } - } catch (const ::parquet::ParquetException&) { - return nullptr; - } catch (const std::exception&) { - return nullptr; - } - } - auto it = column_bloom_filters.find(cache_key); - return it == column_bloom_filters.end() ? nullptr : it->second.get(); - } -}; - -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; -} - -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; -} - -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; -} - -bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { - return false; - } - - const auto& encoding_stats = column_metadata.encoding_stats(); - if (!encoding_stats.empty()) { - bool has_dictionary_data_page = false; - for (const auto& encoding_stat : encoding_stats) { - if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { - continue; - } - if (!is_dictionary_data_encoding(encoding_stat.encoding)) { - return false; - } - has_dictionary_data_page = true; - } - return has_dictionary_data_page; - } - - bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { - if (is_dictionary_data_encoding(encoding)) { - has_dictionary_encoding = true; - continue; - } - if (!is_level_encoding(encoding)) { - return false; - } - } - return has_dictionary_encoding; -} - -bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr) { - return false; - } - if (!column_schema.type_descriptor.is_string_like) { - return false; - } - if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && - column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - return false; - } - return true; -} - -} // namespace - -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words) { - DORIS_CHECK(dict_words != nullptr); - dict_words->clear(); - if (file_reader == nullptr || leaf_column_id < 0) { - return false; - } - - auto row_group_reader = file_reader->RowGroup(row_group_idx); - if (row_group_reader == nullptr) { - return false; - } - auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); - if (page_reader == nullptr) { - return false; - } - - std::shared_ptr<::parquet::Page> page; - try { - page = page_reader->NextPage(); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { - return false; - } - const auto* dictionary_page = static_cast(page.get()); - if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && - dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { - return false; - } - const int32_t dictionary_length = dictionary_page->num_values(); - if (dictionary_length <= 0) { - return false; - } - const auto* dictionary_data = dictionary_page->data(); - const int dictionary_size = dictionary_page->size(); - - dict_words->values.reserve(static_cast(dictionary_length)); - if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { - auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( - ::parquet::Encoding::PLAIN, column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); - if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(byte_array_values[dict_idx].ptr), - byte_array_values[dict_idx].len); - } - dict_words->build_refs(); - return true; - } - if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, - column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::FixedLenByteArray> flba_values( - static_cast(dictionary_length)); - if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(flba_values[dict_idx].ptr), type_length); - } - dict_words->build_refs(); - return true; - } - return false; -} - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { - std::vector fields; - fields.reserve(dict_words.refs.size()); - for (const auto& ref : dict_words.refs) { - fields.push_back(Field::create_field(String(ref.data, ref.size))); - } - return fields; -} - -namespace { - const ParquetColumnSchema* resolve_local_leaf_schema( const std::vector>& schema, const format::LocalColumnId file_column_id) { @@ -762,164 +533,112 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats } // namespace +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state); +} + std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( const ParquetColumnStatistics& statistics) { return make_zonemap_from_statistics(statistics); } ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone) { ParquetColumnStatistics result; - if (statistics == nullptr) { + if (statistics == nullptr || column_value_count < 0) { return result; } - result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; - result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); - result.has_null_count = statistics->HasNullCount(); - if (!result.has_not_null || !statistics->HasMinMax()) { + if (statistics->__isset.null_count && statistics->null_count > column_value_count) { + // An impossible null count makes all derived min/max and all-null flags untrustworthy; + // disable pruning instead of turning corrupt footer metadata into false negatives. return result; } - DORIS_CHECK(column_schema.type != nullptr); - switch (statistics->physical_type()) { - case ::parquet::Type::BOOLEAN: - result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( - statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); - return result; - case ::parquet::Type::INT32: - result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::INT64: - result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::FLOAT: - result.has_min_max = set_decoded_min_max<::parquet::FloatType>( - statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); - return result; - case ::parquet::Type::DOUBLE: - result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( - statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); - return result; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); - return result; - default: - return result; + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; + const int64_t null_count = has_null_count ? statistics->null_count : 0; + const bool has_not_null = has_null_count ? column_value_count > null_count : true; + const std::string* min_value = statistics->__isset.min_value + ? &statistics->min_value + : (statistics->__isset.min ? &statistics->min : nullptr); + const std::string* max_value = statistics->__isset.max_value + ? &statistics->max_value + : (statistics->__isset.max ? &statistics->max : nullptr); + + tparquet::ColumnIndex index; + index.__set_null_pages({!has_not_null}); + index.__set_null_counts({null_count}); + if (min_value != nullptr && max_value != nullptr) { + index.__set_min_values({*min_value}); + index.__set_max_values({*max_value}); + } + // Footer statistics and page indexes share the same little-endian physical encoding. Reusing + // one decoder keeps native row-group and page pruning identical for logical types and NaNs. + if (!build_native_page_statistics(index, column_schema, 0, column_value_count, &result, + timezone)) { + return {}; + } + if (!has_null_count) { + result.has_null_count = false; + result.has_null = true; } + return result; } -bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, - int slot_index, const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); +bool ParquetStatisticsUtils::NativeBloomFilterExcludes( + const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + NativeParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; } namespace { -ParquetRowGroupPruneReason dictionary_prune_reason( - const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request) { - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_dictionary_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_dictionary_pruning(*column_schema, *column_chunk) || - !is_dictionary_encoded_chunk(*column_chunk)) { - continue; - } - - ParquetDictionaryWords dict_words; - if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, - *column_schema, &dict_words)) { - continue; - } - DictionaryEvalContext ctx; - ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { - .data_type = column_schema->type, - .values = dictionary_fields_from_words(dict_words), - }); - if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch) { - return ParquetRowGroupPruneReason::DICTIONARY; +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); } + return; } - return ParquetRowGroupPruneReason::NONE; -} - -ParquetRowGroupPruneReason bloom_filter_prune_reason( - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_cache == nullptr) { - return ParquetRowGroupPruneReason::NONE; - } - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr || - !bloom_filter_supported(*column_schema)) { - continue; - } - auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, - pruning_stats); - if (bloom_filter == nullptr) { + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { continue; } - if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, - *bloom_filter)) { - return ParquetRowGroupPruneReason::BLOOM_FILTER; - } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); } - return ParquetRowGroupPruneReason::NONE; } -void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, - RowGroupBloomFilterCache* bloom_filter_cache) { - DORIS_CHECK(bloom_filter_cache != nullptr); - if (!enable_bloom_filter || file_reader == nullptr) { - return; - } - try { - bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); - } catch (const ::parquet::ParquetException&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } catch (const std::exception&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; } -bool check_statistics(const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { +bool check_native_statistics(const tparquet::FileMetaData& metadata, + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); if (slot_indexes.empty()) { return false; } - ZoneMapEvalContext ctx; for (const int slot_index : slot_indexes) { const auto file_column_id = file_column_id_by_block_position(request, slot_index); @@ -927,270 +646,320 @@ bool check_statistics(const ::parquet::RowGroupMetaData& row_group, continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } - + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; std::shared_ptr zone_map; - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk != nullptr) { + if (chunk.__isset.meta_data) { + const auto& column_metadata = chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + column_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order(metadata, + column_schema->leaf_column_id)); + } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, column_chunk->statistics(), timezone)); + *column_schema, + safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, timezone)); } add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); } - const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); accumulate_zonemap_stats(ctx, pruning_stats); return result == ZoneMapFilterResult::kNoMatch; } -Status select_row_groups_by_metadata_impl( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t row_group_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink - : &pruning_stats->row_group_filter_time); - if (selected_row_groups == nullptr) { - return Status::InvalidArgument("selected_row_groups is null"); - } - selected_row_groups->clear(); +bool is_native_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; +} - const int num_row_groups = metadata.num_row_groups(); - const auto candidate_size = candidate_row_groups == nullptr - ? static_cast(num_row_groups) - : candidate_row_groups->size(); - if (pruning_stats != nullptr) { - // Scan-range ownership is decided before metadata pruning. Count only row groups owned by - // this split so a file divided into multiple splits does not report the full-file total and - // out-of-split groups once per split. - pruning_stats->total_row_groups = cast_set(candidate_size); +bool is_native_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; +} + +bool is_native_dictionary_encoded_chunk(const tparquet::ColumnMetaData& metadata) { + if (!metadata.__isset.dictionary_page_offset || metadata.dictionary_page_offset < 0) { + return false; } - selected_row_groups->reserve(candidate_size); - RowGroupBloomFilterCache bloom_filter_cache; - init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); - for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { - const int row_group_idx = candidate_row_groups == nullptr - ? static_cast(candidate_idx) - : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(row_group_idx < num_row_groups); - auto row_group = metadata.RowGroup(row_group_idx); - if (row_group == nullptr) { - selected_row_groups->push_back(row_group_idx); - continue; + if (metadata.__isset.encoding_stats && !metadata.encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : metadata.encoding_stats) { + if ((encoding_stat.page_type != tparquet::PageType::DATA_PAGE && + encoding_stat.page_type != tparquet::PageType::DATA_PAGE_V2) || + encoding_stat.count <= 0) { + continue; + } + if (!is_native_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; } - ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { - prune_reason = ParquetRowGroupPruneReason::STATISTICS; + return has_dictionary_data_page; + } + bool has_dictionary_encoding = false; + for (const auto encoding : metadata.encodings) { + if (is_native_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + } else if (!is_native_level_encoding(encoding)) { + return false; } + } + return has_dictionary_encoding; +} - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, - file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, - &bloom_filter_cache, pruning_stats); - } +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + for (const auto& projection : request.predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - - if (prune_reason != ParquetRowGroupPruneReason::NONE) { - if (pruning_stats != nullptr) { - pruning_stats->filtered_group_rows += row_group->num_rows(); - if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { - ++pruning_stats->filtered_row_groups_by_statistics; - } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { - ++pruning_stats->filtered_row_groups_by_dictionary; - } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { - ++pruning_stats->filtered_row_groups_by_bloom_filter; - } - } - continue; + } + for (const auto& projection : request.non_predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - selected_row_groups->push_back(row_group_idx); } - return Status::OK(); + return nullptr; } -} // namespace - -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +ParquetRowGroupPruneReason native_dictionary_prune_reason( + const tparquet::RowGroup& row_group, int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - return select_row_groups_by_metadata_impl( - metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, - enable_bloom_filter, pruning_stats, timezone, runtime_state); -} - -namespace { - -template -bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - const auto typed_index = - std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + const format::FileScanRequest& request, const cctz::time_zone* timezone, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile) { + if (file_context == nullptr || file_context->native_metadata == nullptr) { + return ParquetRowGroupPruneReason::NONE; } - const auto& min_value = typed_index->min_values()[page_idx]; - const auto& max_value = typed_index->max_values()[page_idx]; - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + const auto* projection = find_request_projection(request, *file_column_id); + if (column_schema == nullptr || projection == nullptr || column_schema->type == nullptr || + !column_schema->type_descriptor.is_string_like || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + if (!native_metadata_predicate_is_type_safe(*column_schema)) { + // The file-local VARBINARY may feed a table-side STRING cast. Pruning before that cast + // can compare different Field kinds and incorrectly discard a matching row group. + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data || + (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && + chunk.meta_data.type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) || + !is_native_dictionary_encoded_chunk(chunk.meta_data)) { + continue; + } + std::unique_ptr reader; + const std::vector ranges {{0, row_group.num_rows}}; + const std::unordered_map offset_indexes; + // Metadata pruning uses the real native reader, so its page work must be attributed to the + // scan profile even when the row group is eliminated before execution readers are built. + const auto status = NativeColumnReader::create( + *column_schema, projection, file_context->native_file, + file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, + file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, + file_context->native_page_cache_file_key, true, column_reader_profile, &reader); + if (!status.ok() || reader == nullptr) { + continue; + } + auto dictionary_result = reader->dictionary_values(); + if (!dictionary_result.has_value()) { + continue; + } + auto dictionary = std::move(dictionary_result).value(); + std::vector values(dictionary->size()); + for (size_t value_idx = 0; value_idx < dictionary->size(); ++value_idx) { + dictionary->get(value_idx, values[value_idx]); + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = std::move(values), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; } } - if (!valid_min_max(min_value, max_value)) { - // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page - // conservatively by returning usable null-count statistics with has_min_max=false, while - // allowing later pages with finite bounds to remain eligible for pruning. - return true; - } - if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, - timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; + return ParquetRowGroupPruneReason::NONE; } -bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - const auto typed_index = - std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); - const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; +ParquetRowGroupPruneReason native_bloom_filter_prune_reason( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetFileContext* file_context, + ParquetPruningStats* pruning_stats) { + if (file_context == nullptr || file_context->native_file == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !bloom_filter_supported(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; } - page_statistics->has_min_max = true; - return true; - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; } - const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + std::unique_ptr bloom_filter; + Status status; + { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->bloom_filter_read_time); + status = read_native_bloom_filter(chunk.meta_data, file_context->native_file, + file_context->native_io_ctx, &bloom_filter); } - const std::string min( - reinterpret_cast(typed_index->min_values()[page_idx].ptr), - type_length); - const std::string max( - reinterpret_cast(typed_index->max_values()[page_idx].ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; + if (!status.ok() || bloom_filter == nullptr) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + if (ParquetStatisticsUtils::NativeBloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; } - page_statistics->has_min_max = true; - return true; - } - default: - return false; } + return ParquetRowGroupPruneReason::NONE; } -bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BOOLEAN: - return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, - page_idx, DecodedValueKind::BOOL, - page_statistics, timezone); - case ::parquet::Type::INT32: - return set_page_decoded_min_max<::parquet::Int32Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: - return set_page_decoded_min_max<::parquet::Int64Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: - return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, - DecodedValueKind::FLOAT, - page_statistics, timezone); - case ::parquet::Type::DOUBLE: - return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, - page_idx, DecodedValueKind::DOUBLE, - page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, - timezone); - default: - return false; +int64_t native_requested_compressed_bytes( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (chunk.__isset.meta_data && chunk.meta_data.total_compressed_size > 0) { + bytes += chunk.meta_data.total_compressed_size; + } } + return bytes; } -bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - DORIS_CHECK(page_statistics != nullptr); - *page_statistics = ParquetColumnStatistics {}; +} // namespace - const auto& null_pages = column_index->null_pages(); - if (!column_index->has_null_counts() || page_idx >= null_pages.size() || - page_idx >= column_index->null_counts().size()) { - return false; +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile, + ParquetMetadataProbeMode probe_mode) { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); } - - page_statistics->has_null_count = true; - page_statistics->has_null = column_index->null_counts()[page_idx] > 0; - page_statistics->has_not_null = !null_pages[page_idx]; - if (!page_statistics->has_not_null) { - return true; + selected_row_groups->clear(); + const size_t candidate_size = candidate_row_groups == nullptr ? metadata.row_groups.size() + : candidate_row_groups->size(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = cast_set(candidate_size); } - return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); + selected_row_groups->reserve(candidate_size); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + // Candidate ids originate in external split metadata; a corrupt id must not terminate + // the BE while planning an otherwise recoverable file scan. + return Status::Corruption("Invalid Parquet row group candidate {} for {} row groups", + row_group_idx, metadata.row_groups.size()); + } + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.num_rows == 0) { + // Native metadata probes construct positive row ranges; empty groups contribute no + // rows and must be discarded before dictionary, statistics, or Bloom reader setup. + continue; + } + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY && + has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(metadata, row_group, file_schema, request, pruning_stats, + timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = + native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, + timezone, file_context, column_reader_profile); + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, + file_context, pruning_stats); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + selected_row_groups->push_back(row_group_idx); + continue; + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group.num_rows; + pruning_stats->filtered_bytes += + native_requested_compressed_bytes(row_group, file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + } + return Status::OK(); } +namespace { + std::vector intersect_ranges(const std::vector& left, const std::vector& right) { std::vector result; @@ -1223,19 +992,6 @@ int64_t count_range_rows(const std::vector& ranges) { return rows; } -RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, - int64_t row_group_rows) { - const auto& page_locations = offset_index.page_locations(); - const int64_t start = page_locations[page_idx].first_row_index; - const int64_t end = page_idx + 1 == page_locations.size() - ? row_group_rows - : page_locations[page_idx + 1].first_row_index; - DORIS_CHECK(start >= 0); - DORIS_CHECK(end >= start); - DORIS_CHECK(end <= row_group_rows); - return RowRange {start, end - start}; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; @@ -1250,85 +1006,6 @@ void append_row_range(const RowRange& range, std::vector* ranges) { ranges->push_back(range); } -std::optional< - std::pair, std::shared_ptr<::parquet::OffsetIndex>>> -load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, - const ParquetColumnSchema** column_schema) { - DORIS_CHECK(column_schema != nullptr); - *column_schema = nullptr; - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - return std::nullopt; - } - *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { - return std::nullopt; - } - - try { - auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); - auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); - if (column_index == nullptr || offset_index == nullptr || - column_index->null_pages().size() != offset_index->page_locations().size()) { - return std::nullopt; - } - return std::make_pair(std::move(column_index), std::move(offset_index)); - } catch (const ::parquet::ParquetException&) { - return std::nullopt; - } catch (const std::exception&) { - return std::nullopt; - } -} - -bool select_ranges_for_expr_zonemap( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, - int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - DORIS_CHECK(ranges != nullptr); - if (conjuncts.empty()) { - return false; - } - const ParquetColumnSchema* column_schema = nullptr; - const auto page_indexes = - load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); - if (!page_indexes.has_value()) { - return false; - } - const auto& [column_index, offset_index] = *page_indexes; - - ranges->clear(); - ZoneMapEvalStats page_stats; - const auto page_count = offset_index->page_locations().size(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - ParquetColumnStatistics page_statistics; - if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *column_schema, page_idx, &page_statistics, timezone)) { - ranges->clear(); - return false; - } - - ZoneMapEvalContext ctx; - add_slot_zonemap(&ctx, slot_index, column_schema->type, - ParquetStatisticsUtils::MakeZoneMap(page_statistics)); - const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); - page_stats.merge_page_eval_stats(ctx.stats); - if (result == ZoneMapFilterResult::kNoMatch) { - continue; - } - append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); - } - if (pruning_stats != nullptr) { - pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; - pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; - pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; - } - return true; -} - bool ranges_intersect(const std::vector& ranges, const RowRange& range) { const int64_t range_end = range.start + range.length; for (const auto& selected_range : ranges) { @@ -1388,127 +1065,201 @@ void collect_request_leaf_schemas( } } -bool build_page_skip_plan_for_leaf( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, - int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { - DORIS_CHECK(page_skip_plan != nullptr); - *page_skip_plan = ParquetPageSkipPlan {}; - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || - column_schema.descriptor->max_repetition_level() != 0) { +template +bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != sizeof(ValueType) || + column_index.max_values[page_idx].size() != sizeof(ValueType)) { return false; } - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - try { - offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; + const auto min_value = unaligned_load(column_index.min_values[page_idx].data()); + const auto max_value = unaligned_load(column_index.max_values[page_idx].data()); + if constexpr (std::is_integral_v) { + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMEV2) { + int64_t units_per_day = 0; + switch (column_schema.type_descriptor.time_unit) { + case ParquetTimeUnit::MILLIS: + units_per_day = 86400000; + break; + case ParquetTimeUnit::MICROS: + units_per_day = 86400000000; + break; + case ParquetTimeUnit::NANOS: + units_per_day = 86400000000000; + break; + default: + return false; + } + // TIME statistics are pruning proofs. Validate the raw carrier before rescaling so an + // invalid bound at or beyond 24:00 cannot publish a misleading ZoneMap. + if (min_value < 0 || max_value < 0 || min_value >= units_per_day || + max_value >= units_per_day) { + return false; + } + } } - if (offset_index == nullptr) { + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, kind, min_value, &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, kind, max_value, &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; +} - const auto page_count = offset_index->page_locations().size(); - page_skip_plan->leaf_column_id = column_schema.leaf_column_id; - page_skip_plan->skipped_pages.resize(page_count); - page_skip_plan->skipped_page_compressed_sizes.resize(page_count); - const auto& page_locations = offset_index->page_locations(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); - if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { - continue; - } - page_skip_plan->skipped_pages[page_idx] = 1; - page_skip_plan->skipped_page_compressed_sizes[page_idx] = - page_locations[page_idx].compressed_page_size; - append_row_range(row_range, &page_skip_plan->skipped_ranges); +bool set_native_page_boolean_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != 1 || + column_index.max_values[page_idx].size() != 1) { + return false; } - if (page_skip_plan->empty()) { - *page_skip_plan = ParquetPageSkipPlan {}; + // Parquet BOOLEAN statistics use the same one-bit value representation as PLAIN pages; bits + // outside the value bit are padding and must not change false into true. + const uint8_t min_value = static_cast(column_index.min_values[page_idx][0]) & 1; + const uint8_t max_value = static_cast(column_index.max_values[page_idx][0]) & 1; + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, DecodedValueKind::BOOL, min_value, + &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, DecodedValueKind::BOOL, max_value, + &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } return true; } -void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans) { - DORIS_CHECK(page_skip_plans != nullptr); - page_skip_plans->clear(); - std::vector leaf_schemas; - collect_request_leaf_schemas(file_schema, request, &leaf_schemas); - for (const auto* leaf_schema : leaf_schemas) { - DORIS_CHECK(leaf_schema != nullptr); - ParquetPageSkipPlan page_skip_plan; - if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, - &page_skip_plan)) { - page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = {}; + if (!column_index.__isset.null_counts || page_idx >= column_index.null_pages.size() || + page_idx >= column_index.null_counts.size()) { + return false; + } + const int64_t null_count = column_index.null_counts[page_idx]; + const bool all_null = column_index.null_pages[page_idx]; + if (page_rows < 0 || null_count < 0 || null_count > page_rows || + all_null != (null_count == page_rows)) { + // The caller supplies the exact flat page or row-group span. Contradictory optional null + // metadata must disable pruning instead of turning a partial span into an all-null proof. + return false; + } + page_statistics->has_null_count = true; + page_statistics->has_null = null_count > 0; + page_statistics->has_not_null = !all_null; + if (!page_statistics->has_not_null) { + return true; + } + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BOOLEAN: + return set_native_page_boolean_min_max(column_index, column_schema, page_idx, + page_statistics, timezone); + case tparquet::Type::INT32: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::INT64: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::FLOAT: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, page_statistics, + timezone); + case tparquet::Type::DOUBLE: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::DOUBLE, page_statistics, + timezone); + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (page_idx >= column_index.min_values.size() || + page_idx >= column_index.max_values.size()) { + return false; + } + const auto& min_value = column_index.min_values[page_idx]; + const auto& max_value = column_index.max_values[page_idx]; + const bool fixed = + column_schema.type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (fixed && + (column_schema.type_descriptor.fixed_length <= 0 || + min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || + max_value.size() != static_cast(column_schema.type_descriptor.fixed_length))) { + return false; } + const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; + if (!set_decoded_binary_field(column_schema, kind, + StringRef(min_value.data(), min_value.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, kind, + StringRef(max_value.data(), max_value.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; + } + default: + return false; } } -} // namespace - -bool ParquetStatisticsUtils::TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); +RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& locations = offset_index.page_locations; + const int64_t start = locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == locations.size() ? row_group_rows + : locations[page_idx + 1].first_row_index; + return {.start = start, .length = end - start}; } -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +} // namespace + +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - int64_t page_index_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink + int64_t filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink : &pruning_stats->page_index_filter_time); DORIS_CHECK(selected_ranges != nullptr); selected_ranges->clear(); + selected_ranges->push_back({.start = 0, .length = row_group_rows}); if (page_skip_plans != nullptr) { page_skip_plans->clear(); } - if (row_group_rows <= 0) { - return Status::OK(); - } - selected_ranges->push_back(RowRange {0, row_group_rows}); - if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || - file_reader == nullptr) { - return Status::OK(); - } - - std::shared_ptr<::parquet::PageIndexReader> page_index_reader; - std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; - try { - if (pruning_stats != nullptr) { - ++pruning_stats->page_index_read_calls; - } - { - int64_t read_page_index_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink - : &pruning_stats->read_page_index_time); - page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - row_group_index_reader = page_index_reader->RowGroup(row_group_idx); - } - } catch (const ::parquet::ParquetException&) { - return Status::OK(); - } catch (const std::exception&) { + if (row_group_rows <= 0 || !config::enable_parquet_page_index || + !has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) { return Status::OK(); } - if (row_group_index_reader == nullptr) { - return Status::OK(); + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; } std::map conjuncts_by_slot; @@ -1518,19 +1269,53 @@ Status select_row_group_ranges_by_page_index( conjuncts_by_slot[slot_index].push_back(conjunct); } } - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !detail::has_supported_type_defined_order(metadata, column_schema->leaf_column_id)) { + continue; + } + const auto index_it = page_indexes.find(column_schema->leaf_column_id); + if (index_it == page_indexes.end()) { + continue; + } + const auto& indexes = index_it->second; std::vector filter_ranges; - if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, - slot_index, conjuncts, row_group_rows, &filter_ranges, - pruning_stats, timezone)) { + bool usable = true; + for (size_t page_idx = 0; page_idx < indexes.offset_index.page_locations.size(); + ++page_idx) { + const auto page_range = + native_page_row_range(indexes.offset_index, page_idx, row_group_rows); + ParquetColumnStatistics statistics; + if (!build_native_page_statistics(indexes.column_index, *column_schema, page_idx, + page_range.length, &statistics, timezone)) { + usable = false; + break; + } + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(statistics)); + if (VExprContext::evaluate_zonemap_filter(conjuncts, ctx) != + ZoneMapFilterResult::kNoMatch) { + append_row_range(page_range, &filter_ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += + ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; + } + } + if (!usable) { continue; } *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); if (selected_ranges->empty()) { - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } if (pruning_stats != nullptr) { pruning_stats->filtered_page_rows += row_group_rows; ++pruning_stats->filtered_row_groups_by_page_index; @@ -1538,14 +1323,37 @@ Status select_row_group_ranges_by_page_index( return Status::OK(); } } + if (page_skip_plans != nullptr) { - build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans); + std::vector leaves; + collect_request_leaf_schemas(file_schema, request, &leaves); + for (const auto* leaf : leaves) { + const auto index_it = page_indexes.find(leaf->leaf_column_id); + if (index_it == page_indexes.end() || leaf->max_repetition_level != 0) { + continue; + } + const auto& offset_index = index_it->second.offset_index; + ParquetPageSkipPlan skip_plan; + skip_plan.leaf_column_id = leaf->leaf_column_id; + skip_plan.skipped_pages.resize(offset_index.page_locations.size()); + skip_plan.skipped_page_compressed_sizes.resize(offset_index.page_locations.size()); + for (size_t page_idx = 0; page_idx < offset_index.page_locations.size(); ++page_idx) { + const auto range = native_page_row_range(offset_index, page_idx, row_group_rows); + if (range.length == 0 || ranges_intersect(*selected_ranges, range)) { + continue; + } + skip_plan.skipped_pages[page_idx] = 1; + skip_plan.skipped_page_compressed_sizes[page_idx] = + offset_index.page_locations[page_idx].compressed_page_size; + append_row_range(range, &skip_plan.skipped_ranges); + } + if (!skip_plan.empty()) { + page_skip_plans->emplace(skip_plan.leaf_column_id, std::move(skip_plan)); + } + } } if (pruning_stats != nullptr) { - const int64_t selected_rows = count_range_rows(*selected_ranges); - DORIS_CHECK(selected_rows <= row_group_rows); - pruning_stats->filtered_page_rows += row_group_rows - selected_rows; + pruning_stats->filtered_page_rows += row_group_rows - count_range_rows(*selected_ranges); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 9e94562bf2d37b..72381548656f9d 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include "common/status.h" @@ -27,16 +30,9 @@ #include "core/string_ref.h" #include "exprs/vexpr_fwd.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/selection_vector.h" -namespace parquet { -class BloomFilter; -class ColumnIndex; -class FileMetaData; -class ParquetFileReader; -class Statistics; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -44,6 +40,7 @@ class time_zone; namespace doris { class RuntimeState; namespace segment_v2 { +class BloomFilter; struct ZoneMap; } // namespace segment_v2 } // namespace doris @@ -51,36 +48,21 @@ struct ZoneMap; namespace doris::format::parquet { struct ParquetColumnSchema; - -// ============================================================================ -// ============================================================================ - -struct ParquetDictionaryWords { - std::vector values; - std::vector refs; - - void clear() { - values.clear(); - refs.clear(); - } - - void build_refs() { - refs.clear(); - refs.reserve(values.size()); - for (const auto& value : values) { - refs.emplace_back(value.data(), value.size()); - } - } -}; - -// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied -// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this -// helper so they agree on dictionary id -> Doris string value mapping. -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words); - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); +struct ParquetFileContext; +struct ParquetTypeDescriptor; + +namespace detail { +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size); +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id); +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +} // namespace detail // ============================================================================ // ============================================================================ @@ -93,6 +75,7 @@ struct ParquetPruningStats { int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_bytes = 0; // requested bytes in pruned row groups int64_t filtered_page_rows = 0; // rows pruned by page index int64_t selected_row_ranges = 0; // selected row range count int64_t page_index_read_calls = 0; // Page Index read count @@ -100,6 +83,7 @@ struct ParquetPruningStats { int64_t row_group_filter_time = 0; // row-group pruning time (ns) int64_t page_index_filter_time = 0; // page-index pruning time (ns) int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t parse_page_index_time = 0; // lazy page-index materialization time (ns) int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks @@ -116,6 +100,20 @@ struct ParquetColumnStatistics { bool has_any_statistics() const { return has_null_count || has_min_max; } }; +struct NativeParquetPageIndex { + tparquet::ColumnIndex column_index; + tparquet::OffsetIndex offset_index; +}; + +enum class ParquetMetadataProbeMode { + ALL, + FOOTER_ONLY, + EXPENSIVE_ONLY, +}; + +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state); + // ============================================================================ // ============================================================================ // VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) @@ -128,32 +126,29 @@ struct ParquetStatisticsUtils { const ParquetColumnStatistics& statistics); static ParquetColumnStatistics TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, - const cctz::time_zone* timezone = nullptr); - - static bool TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); - - static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter); + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone = nullptr); + + static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const segment_v2::BloomFilter& bloom_filter); }; Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}, + ParquetMetadataProbeMode probe_mode = ParquetMetadataProbeMode::ALL); -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index 8411d53f0fba91..be5ecd1c8b0772 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -17,340 +17,31 @@ #include "format_v2/parquet/parquet_type.h" -#include - -#include -#include - -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/primitive_type.h" - namespace doris::format::parquet { -namespace { - -DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { - return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); -} - -PrimitiveType decimal_primitive_type(int precision) { - return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; -} - -void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, - ParquetTypeDescriptor* result) { - result->is_decimal = true; - result->decimal_precision = precision; - result->decimal_scale = scale; - switch (column->physical_type()) { - case ::parquet::Type::INT32: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; - break; - case ::parquet::Type::INT64: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; - break; - default: - result->extra_type_info = ParquetExtraTypeInfo::NONE; - break; - } -} - -void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { - result->integer_bit_width = bit_width; - result->is_unsigned_integer = !is_signed; -} - -DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const bool nullable = column->max_definition_level() > 0; - switch (column->converted_type()) { - case ::parquet::ConvertedType::UTF8: - case ::parquet::ConvertedType::ENUM: - case ::parquet::ConvertedType::JSON: - case ::parquet::ConvertedType::BSON: - return create_type(TYPE_STRING, nullable); - case ::parquet::ConvertedType::DECIMAL: - mark_decimal(column, column->type_precision(), column->type_scale(), result); - return create_type(decimal_primitive_type(column->type_precision()), nullable, - column->type_precision(), column->type_scale()); - case ::parquet::ConvertedType::DATE: - return create_type(TYPE_DATEV2, nullable); - case ::parquet::ConvertedType::TIME_MILLIS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIME_MICROS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIMESTAMP_MILLIS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 3); - case ::parquet::ConvertedType::TIMESTAMP_MICROS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 6); - // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: - // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and - // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider - // signed type so all values in the unsigned range can be represented. - case ::parquet::ConvertedType::INT_8: - mark_integer(8, true, result); - return create_type(TYPE_TINYINT, nullable); - case ::parquet::ConvertedType::UINT_8: - mark_integer(8, false, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::INT_16: - mark_integer(16, true, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::UINT_16: - mark_integer(16, false, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::INT_32: - mark_integer(32, true, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::UINT_32: - mark_integer(32, false, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::INT_64: - mark_integer(64, true, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::UINT_64: - mark_integer(64, false, result); - return create_type(TYPE_LARGEINT, nullable); - case ::parquet::ConvertedType::NONE: - default: - return nullptr; - } -} - -DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const auto& logical_type = column->logical_type(); - if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { - return nullptr; - } - const bool nullable = column->max_definition_level() > 0; - if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || - logical_type->is_BSON() || logical_type->is_UUID()) { - return create_type(TYPE_STRING, nullable); - } - if (logical_type->is_decimal()) { - const auto& decimal_type = static_cast(*logical_type); - mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); - return create_type(decimal_primitive_type(decimal_type.precision()), nullable, - decimal_type.precision(), decimal_type.scale()); - } - if (logical_type->is_date()) { - return create_type(TYPE_DATEV2, nullable); - } - if (logical_type->is_time()) { - const auto& time_type = static_cast(*logical_type); - if (time_type.is_adjusted_to_utc()) { - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - } - int scale = 0; - if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else { - return nullptr; - } - return create_type(TYPE_TIMEV2, nullable, 0, scale); - } - if (logical_type->is_timestamp()) { - const auto& timestamp_type = - static_cast(*logical_type); - int scale = 0; - if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { - scale = 6; - result->time_unit = ParquetTimeUnit::NANOS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; - } else { - return nullptr; - } - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); - return create_type(TYPE_DATETIMEV2, nullable, 0, scale); - } - if (logical_type->is_int()) { - const auto& int_type = static_cast(*logical_type); - mark_integer(int_type.bit_width(), int_type.is_signed(), result); - switch (int_type.bit_width()) { - case 8: - return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); - case 16: - return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); - case 32: - return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); - case 64: - return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); - default: - return nullptr; - } - } - if (logical_type->is_float16()) { - if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || - column->type_length() != 2) { - return nullptr; - } - result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; - return create_type(TYPE_FLOAT, nullable); - } - return nullptr; -} - -DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { - const bool nullable = column->max_definition_level() > 0; - DataTypePtr type; - switch (column->physical_type()) { - case ::parquet::Type::BOOLEAN: - type = std::make_shared(); - break; - case ::parquet::Type::INT32: - type = std::make_shared(); - break; - case ::parquet::Type::INT64: - type = std::make_shared(); - break; - case ::parquet::Type::FLOAT: - type = std::make_shared(); - break; - case ::parquet::Type::DOUBLE: - type = std::make_shared(); - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - type = std::make_shared(); - break; - case ::parquet::Type::INT96: - type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); - break; - default: - return nullptr; - } - return nullable ? make_nullable(type) : type; -} - -bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { - switch (physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::INT96: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return true; - default: - return false; - } -} - -} // namespace - -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { - if (column == nullptr) { - return {}; - } - auto path = column->path(); - if (path) { - return path->ToDotString(); - } - return column->name(); -} - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { - ParquetTypeDescriptor result; - if (column == nullptr) { - return result; - } - - result.physical_type = column->physical_type(); - result.converted_type = column->converted_type(); - result.fixed_length = column->type_length(); - result.physical_doris_type = physical_type_to_doris_type(column); - - if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { - result.doris_type = logical_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else if (auto converted_type = converted_type_to_doris_type(column, &result); - converted_type != nullptr) { - result.doris_type = converted_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else { - result.doris_type = result.physical_doris_type; - if (result.physical_type == ::parquet::Type::INT96) { - result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; - } - } - - result.is_string_like = !result.is_decimal && - result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result.physical_type == ::parquet::Type::BYTE_ARRAY || - result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - - if (!record_reader_physical_type_supported(result.physical_type)) { - result.supports_record_reader = false; - } - return result; -} - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { - return type_descriptor.supports_record_reader; -} DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return DecodedValueKind::BOOL; - case ::parquet::Type::INT32: + case tparquet::Type::INT32: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { return DecodedValueKind::UINT32; } return DecodedValueKind::INT32; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { return DecodedValueKind::UINT64; } return DecodedValueKind::INT64; - case ::parquet::Type::INT96: + case tparquet::Type::INT96: return DecodedValueKind::INT96; - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return DecodedValueKind::FLOAT; - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return DecodedValueKind::DOUBLE; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return DecodedValueKind::FIXED_BINARY; - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: default: return DecodedValueKind::BINARY; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index a4d99abc0e982a..7ab3d2b3b39d8d 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -15,17 +15,13 @@ #pragma once -#include +#include #include #include "core/data_type/data_type.h" #include "core/data_type_serde/decoded_column_view.h" -namespace parquet { -class ColumnDescriptor; -} // namespace parquet - namespace doris::format::parquet { // ============================================================================ @@ -53,14 +49,15 @@ enum class ParquetTimeUnit { // ============================================================================ // ============================================================================ struct ParquetTypeDescriptor { + // Keep the V2 semantic tree on generated Thrift enums; using parquet-cpp descriptors here + // would reintroduce a second metadata model and make native planning bypassable. DataTypePtr doris_type; // Physical fallback used only to keep file schema construction alive when the logical type is // unsupported. Column reader creation still rejects unsupported_reason before decoding. DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; - ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + tparquet::Type::type physical_type = tparquet::Type::INT32; int integer_bit_width = -1; // bit width for INT_8/16/32/64 int decimal_precision = -1; // precision for DECIMAL(p,s) int decimal_scale = -1; // scale for DECIMAL(p,s) @@ -70,16 +67,9 @@ struct ParquetTypeDescriptor { bool is_timestamp = false; // whether this is a timestamp type bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 - bool supports_record_reader = true; // whether Arrow RecordReader can read this type std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 352fbbd7c3d215..db5def75b8c8a7 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -15,145 +15,25 @@ #include "format_v2/parquet/reader/column_reader.h" -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include -#include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { -namespace { - -class DataPageSkipFilter { -public: - DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, - ParquetPageSkipProfile page_skip_profile) - : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { - DORIS_CHECK(_page_skip_plan != nullptr); - } - - bool operator()(const ::parquet::DataPageStats&) { - // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for - // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. - const size_t page_idx = _next_data_page_idx++; - const bool skip = _page_skip_plan->should_skip_page(page_idx); - if (!skip) { - return false; - } - update_skip_profile(page_idx); - return true; - } - -private: - void update_skip_profile(size_t page_idx) const { - if (_page_skip_profile.skipped_pages != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); - } - if (_page_skip_profile.skipped_bytes != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_bytes, - _page_skip_plan->skipped_page_compressed_size(page_idx)); - } - } - - const ParquetPageSkipPlan* _page_skip_plan = nullptr; - ParquetPageSkipProfile _page_skip_profile; - size_t _next_data_page_idx = 0; -}; - -const ParquetPageSkipPlan* find_page_skip_plan( - const std::map* page_skip_plans, int leaf_column_id) { - if (page_skip_plans == nullptr) { - return nullptr; - } - const auto plan_it = page_skip_plans->find(leaf_column_id); - return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; -} - -void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, - const std::map* page_skip_plans, - int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { - DORIS_CHECK(page_reader != nullptr); - const ParquetPageSkipPlan* page_skip_plan = - find_page_skip_plan(page_skip_plans, leaf_column_id); - if (page_skip_plan == nullptr) { - return; - } - page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); -} - -bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { - if (column_schema.type_descriptor.supports_record_reader) { - return true; - } - const auto& type_descriptor = column_schema.type_descriptor; - if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && - type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || - type_descriptor.is_decimal || type_descriptor.is_timestamp || - type_descriptor.is_string_like) { - return false; - } - if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && - type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { - return false; - } - switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - return true; - default: - return false; - } - return true; -} -} // namespace +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _type(std::move(type)), + _name(schema.name) {} Status ParquetColumnReader::skip(int64_t rows) { return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); } -void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - int64_t child_cursor = nested_build_level_cursor(); - const auto& child_rep_levels = nested_repetition_levels(); - const int64_t child_levels_written = nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { - break; - } - } - set_nested_build_level_cursor(child_cursor); -} - void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, rows); @@ -166,21 +46,16 @@ void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { } } -Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, +Status ParquetColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Parquet selected read result is null for column {}", - name()); - } - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + DORIS_CHECK(column); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); - const auto ranges = selection_to_ranges(sel, selected_rows); + const auto ranges = selection_to_ranges(selection, selected_rows); int64_t cursor = 0; for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", - range.start, range.start + range.length, name()); - } + DORIS_CHECK(range.start >= cursor); + DORIS_CHECK(range.start + range.length <= batch_rows); RETURN_IF_ERROR(skip(range.start - cursor)); int64_t range_rows_read = 0; @@ -206,453 +81,15 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } -ParquetColumnReaderFactory::ParquetColumnReaderFactory( - std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, - const std::map* page_skip_plans, - ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) - : _row_group(std::move(row_group)), - _record_readers(static_cast(num_leaf_columns)), - _dictionary_record_readers(static_cast(num_leaf_columns)), - _page_skip_plans(page_skip_plans), - _page_skip_profile(page_skip_profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _column_reader_profile(column_reader_profile) {} - -std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( - int64_t row_group_first_row) const { - return std::make_unique(row_group_first_row, _column_reader_profile); -} - -std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { - return std::make_unique(context, row_group_first_row, - _column_reader_profile); -} - -Status ParquetColumnReaderFactory::make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - const auto* page_skip_plan = - use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) - : nullptr; - *reader = std::make_unique(column_schema, std::move(record_reader), - page_skip_plan, _timezone, _enable_strict_mode, - _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_scalar_column_reader( - const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { - return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", - column_schema.name); - } - if (column_schema.leaf_column_id < 0 || - column_schema.leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - column_schema.leaf_column_id, column_schema.name); - } - if (column_schema.descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", - column_schema.name); - } - if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || - column_schema.descriptor->max_definition_level() > 1)) { - return Status::NotSupported( - "Current parquet scalar reader only supports flat primitive columns; column {} is " - "not supported", - column_schema.name); - } - if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { - return Status::NotSupported( - "Current parquet nested scalar reader does not support column {}", - column_schema.name); - } - if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { - return Status::NotSupported("Current parquet scalar reader does not support column {}", - column_schema.name); - } - std::shared_ptr<::parquet::internal::RecordReader> record_reader; - // Nested readers implement skip() by materializing rows into a scratch column. If Arrow - // page filtering is also installed, those scratch reads can consume the next selected row - // after a page-index range gap. Keep page filtering on flat scalar readers only. - RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, - column_schema.name, !is_nested, read_dictionary, - &record_reader)); - return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); -} - -// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader -Status ParquetColumnReaderFactory::get_record_reader( - int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, - bool install_page_filter, bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (_row_group == nullptr) { - return Status::InternalError("Parquet row group reader is not initialized for column {}", - name); - } - if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - leaf_column_id, name); - } - if (descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); - } - auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; - if (record_readers[leaf_column_id] == nullptr) { - try { - auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); - if (install_page_filter) { - install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, - _page_skip_profile); - } - const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); - record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( - descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/read_dictionary, - /*read_dense_for_nullable=*/false); - record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } - } - if (record_readers[leaf_column_id] == nullptr) { - return Status::Corruption("Failed to create parquet record reader for column {}", name); - } - *reader = record_readers[leaf_column_id]; - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_struct_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - std::vector> child_readers; - child_readers.reserve(column_schema.children.size()); - std::vector child_output_indices; - child_output_indices.reserve(column_schema.children.size()); - DataTypes projected_child_types; - Strings projected_child_names; - for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { - const auto& child_schema = column_schema.children[child_idx]; - const auto* child_projection = - format::find_child_projection(projection, child_schema->local_id); - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - std::unique_ptr child_reader; - RETURN_IF_ERROR( - create_column_reader(*child_schema, child_projection, true, false, &child_reader)); - child_output_indices.push_back(static_cast(projected_child_types.size())); - projected_child_types.push_back(make_nullable(child_reader->type())); - projected_child_names.push_back(child_reader->name()); - child_readers.push_back(std::move(child_reader)); - } - if (format::is_partial_projection(projection) && - projected_child_types.size() != projection->children.size()) { - return Status::InvalidArgument( - "Parquet STRUCT projection for column {} contains invalid child", - column_schema.name); - } - if (projected_child_types.empty() && !column_schema.children.empty()) { - return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", - column_schema.name); - } - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(projection)) { - type = std::make_shared(projected_child_types, projected_child_names); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique( - column_schema, std::move(type), std::move(child_readers), - std::move(child_output_indices), _column_reader_profile); +Status ParquetColumnReader::select_with_fixed_width_filter(const SelectionVector&, uint16_t, + int64_t, const VExprSPtrs&, int, + IColumn*, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *used_filter = false; return Status::OK(); } -Status ParquetColumnReaderFactory::create_list_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for column {}", - column_schema.name); - } - std::unique_ptr element_reader; - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - if (format::is_partial_projection(projection) && element_projection == nullptr) { - return Status::NotSupported("Parquet LIST projection for column {} contains no element", - column_schema.name); - } - RETURN_IF_ERROR( - create_column_reader(element_schema, element_projection, true, false, &element_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(element_projection)) { - type = std::make_shared(element_reader->type()); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique(column_schema, std::move(type), - std::move(element_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_map_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP layout for column {}", - column_schema.name); - } - const auto& key_schema = *column_schema.children[0]; - const auto& value_schema = *column_schema.children[1]; - const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); - if (format::is_partial_projection(projection)) { - if (value_projection == nullptr) { - return Status::NotSupported("Parquet MAP projection for column {} contains no value", - column_schema.name); - } - for (const auto& child_projection : projection->children) { - if (child_projection.local_id() == key_schema.local_id) { - continue; - } - if (child_projection.local_id() != value_schema.local_id) { - return Status::InvalidArgument( - "Parquet MAP projection for column {} contains invalid child", - column_schema.name); - } - } - } - std::unique_ptr key_reader; - // MAP materialization always needs the full key stream. It owns entry existence, offsets and - // key equality semantics, so MAP projection is defined only as value-subtree pruning. - RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); - std::unique_ptr value_reader; - RETURN_IF_ERROR( - create_column_reader(value_schema, value_projection, true, false, &value_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(value_projection)) { - type = std::make_shared(make_nullable(key_reader->type()), - make_nullable(value_reader->type())); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = - std::make_unique(column_schema, std::move(type), std::move(key_reader), - std::move(value_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, - bool read_dictionary) const { - return create_column_reader(column_schema, projection, false, read_dictionary, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - return create_count_shape_reader_impl(column_schema, projection, false, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader_impl( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, is_nested, false, reader); - case ParquetColumnSchemaKind::STRUCT: { - if (column_schema.children.empty()) { - return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", - column_schema.name); - } - const ParquetColumnSchema* child_schema = nullptr; - const format::LocalColumnIndex* child_projection = nullptr; - if (format::is_partial_projection(projection)) { - const auto child_id = projection->children[0].local_id(); - const auto child_it = std::ranges::find_if( - column_schema.children, - [&](const auto& child) { return child->local_id == child_id; }); - if (child_it == column_schema.children.end()) { - return Status::InvalidArgument( - "Parquet COUNT projection for column {} contains invalid child", - column_schema.name); - } - child_schema = child_it->get(); - child_projection = &projection->children[0]; - } else { - child_schema = column_schema.children[0].get(); - } - DORIS_CHECK(child_schema != nullptr); - return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); - } - case ParquetColumnSchemaKind::LIST: { - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", - column_schema.name); - } - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - return create_count_shape_reader_impl(element_schema, element_projection, true, reader); - } - case ParquetColumnSchemaKind::MAP: { - if (column_schema.children.empty()) { - return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", - column_schema.name); - } - // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness - // from it avoids creating a value reader, which is the expensive path for files with huge - // MAP value strings. - return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); - } - } - return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", - column_schema.name); -} - -Status ParquetColumnReaderFactory::create_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (is_nested) { - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, true, false, reader); - } - return create_scalar_column_reader(column_schema, false, read_dictionary, reader); - case ParquetColumnSchemaKind::STRUCT: - return create_struct_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::LIST: - return create_list_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::MAP: - return create_map_column_reader(column_schema, projection, reader); - } - return Status::NotSupported("Unsupported parquet column schema kind for column {}", - column_schema.name); -} - -ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, - ParquetColumnReaderProfile profile) - : _profile(profile), - _field_id(schema.local_id), - _leaf_column_id(schema.leaf_column_id), - _nullable_definition_level(schema.nullable_definition_level), - _repeated_repetition_level(schema.repeated_repetition_level), - _definition_level(schema.definition_level), - _repetition_level(schema.repetition_level), - _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), - _type(std::move(type)), - _name(schema.name) {} - -Status ParquetColumnReader::load_nested_batch(int64_t) { - return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); -} - -Status ParquetColumnReader::load_nested_levels_batch(int64_t) { - return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", - _name); -} - -Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { - return Status::NotSupported("Parquet nested column build is not supported for column {}", - _name); -} - -Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { - return Status::NotSupported("Parquet nested column consume is not supported for column {}", - _name); -} - -Status ParquetColumnReader::skip_nested_rows(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - // A nested parent row may expand to many child values. Capping the number of parent rows per - // loaded batch bounds that amplification for large holes. The consume interface advances the - // loaded definition/repetition levels recursively without constructing a discarded Column. - constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; - int64_t remaining_rows = rows; - while (remaining_rows > 0) { - const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); - RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); - int64_t rows_consumed = 0; - RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); - if (rows_consumed != batch_rows) { - return Status::Corruption( - "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", - _name, rows_consumed, batch_rows); - } - remaining_rows -= batch_rows; - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -const std::vector& ParquetColumnReader::nested_definition_levels() const { - static const std::vector empty; - return empty; -} - -const std::vector& ParquetColumnReader::nested_repetition_levels() const { - static const std::vector empty; - return empty; -} - -int64_t ParquetColumnReader::nested_levels_written() const { - return 0; -} - -bool ParquetColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 51dbd44c11c226..6914a5be7d165d 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -16,216 +16,81 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" -#include "core/column/column_nullable.h" +#include "core/column/column.h" #include "core/data_type/data_type.h" -#include "format_v2/column_data.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/runtime_profile.h" - -namespace parquet { -class ColumnDescriptor; -class RowGroupReader; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris { -class IColumn; -} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; +// Scan-time column contract for FileScannerV2. +// +// Physical Parquet columns are implemented by NativeColumnReader, which owns Doris' native page +// decoder and writes directly into the destination Doris column. Synthetic row-position/global-id +// readers implement the same cursor contract. Arrow RecordReader, ParquetLeafBatch, decoded value +// views, and the former nested build/consume protocol intentionally do not belong to this API. class ParquetColumnReader { public: virtual ~ParquetColumnReader() = default; virtual int file_column_id() const { return _field_id; } - virtual int parquet_leaf_column_id() const { return _leaf_column_id; } - - int16_t nullable_definition_level() const { return _nullable_definition_level; } - int16_t repeated_repetition_level() const { return _repeated_repetition_level; } - virtual const DataTypePtr& type() const { return _type; } virtual const std::string& name() const { return _name; } const ParquetColumnReaderProfile& profile() const { return _profile; } + // Consume rows from the row-group cursor and append them to column. virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + // Consume rows without materializing values. virtual Status skip(int64_t rows); - virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, - MutableColumnPtr& column); + // Consume batch_rows and append only selection[0, selected_rows). The default implementation + // coalesces adjacent indices into ranges; native readers override it with one decoder call. + virtual Status select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column); - virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, + virtual Status select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); - virtual Status load_nested_batch(int64_t rows); - - // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that - // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are - // available; value indices and payload columns may be absent. Callers may inspect the levels or - // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, - // skipping ARRAY uses this method to find ARRAY boundaries without constructing a - // ColumnString. The underlying Arrow reader may still decode a page because it has no public - // levels-only API. Normal scans that need output values use load_nested_batch() instead. - virtual Status load_nested_levels_batch(int64_t rows); - - virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read); - - // Consume logical values from a batch previously loaded by load_nested_batch() or - // load_nested_levels_batch() without appending them to an output Column. Implementations must - // advance exactly the same nested level cursors and perform the same shape/null/alignment - // validation as build_nested_column(). The levels-only form is preferred for skip paths because - // it avoids transferring leaf payloads into Doris Columns when they will be discarded. - // - // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf - // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY - // rows but only two element values. A MAP implementation must also consume key/value streams - // in lockstep, while a nullable STRUCT consumes no child value for a null parent. - // - // Callers must not use the ordinary skip() after either load call: the leaf stream has already - // advanced into an in-memory nested batch, and doing so would advance it twice. - // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. - virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); - - virtual const std::vector& nested_definition_levels() const; - virtual const std::vector& nested_repetition_levels() const; - virtual int64_t nested_levels_written() const; - virtual bool is_or_has_repeated_child() const; - virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); - - int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } - void set_nested_build_level_cursor(int64_t cursor) { - DORIS_CHECK(cursor >= 0); - _nested_build_level_cursor = cursor; + // Consume batch_rows and evaluate eligible fixed-width values without first constructing a + // complete predicate column. Append survivors when projected_column is non-null. Implementations + // must leave the cursor untouched when used_filter=false. + virtual Status select_with_fixed_width_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + + // Native statistics are cumulative and can be recursively aggregated for complex columns. + // Flush once at the scheduler batch boundary instead of snapshotting after each operation. + virtual void flush_profile() {} + virtual bool crossed_page_since_last_batch() { return false; } + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } - void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } protected: - ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); ParquetColumnReader() = default; - // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level - // memory when a parent expands to many children; the levels-only load plus - // consume_nested_column() avoids payload materialization and output Columns. - Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; void update_reader_skip_rows(int64_t rows) const; ParquetColumnReaderProfile _profile; - const int _field_id = -1; // child ordinal in the parent node - const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) - const int16_t _nullable_definition_level = - 0; // definition-level threshold where this node becomes nullable - const int16_t _repeated_repetition_level = - 0; // repetition level of the nearest repeated ancestor - const int16_t _definition_level = 0; // definition level accumulated to this node - const int16_t _repetition_level = 0; // repetition level accumulated to this node - const int16_t _repeated_ancestor_definition_level = - 0; // definition level of the nearest repeated ancestor - const DataTypePtr _type; // Doris target type - const std::string _name; // column name for error messages - int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) + const int _field_id = -1; + const int _leaf_column_id = -1; + const DataTypePtr _type; + const std::string _name; }; -class ParquetColumnReaderFactory { -public: - ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, - int num_leaf_columns, - const std::map* page_skip_plans = nullptr, - ParquetPageSkipProfile page_skip_profile = {}, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false, - ParquetColumnReaderProfile column_reader_profile = {}); - - Status create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, bool read_dictionary = false) const; - - // Create a scalar reader for one representative leaf that carries the top-level column shape. - // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether - // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately - // uses the key leaf because the key stream owns entry existence and avoids reading value pages. - Status create_count_shape_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create(const ParquetColumnSchema& column_schema, - std::unique_ptr* reader) const { - return create(column_schema, nullptr, reader); - } - - std::unique_ptr create_row_position_column_reader( - int64_t row_group_first_row) const; - std::unique_ptr create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; - -private: - Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - - Status create_struct_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_list_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_map_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - bool is_nested, - std::unique_ptr* reader) const; - - Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, - const std::string& name, bool install_page_filter, - bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const; - - Status make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - bool use_page_skip_plan, std::unique_ptr* reader) const; - - std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader - mutable std::vector> - _record_readers; // RecordReader cache by leaf_column_id - mutable std::vector> - _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id - const std::map* _page_skip_plans = - nullptr; // page-index pruning result - ParquetPageSkipProfile _page_skip_profile; // page skip profile - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - ParquetColumnReaderProfile _column_reader_profile; // column reader profile -}; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp new file mode 100644 index 00000000000000..171107107f1a38 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/count_column_reader.h" + +#include +#include +#include + +#include "common/config.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +Status find_count_leaf(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const ParquetColumnSchema** leaf) { + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + schema.name); + } + *leaf = &schema; + return Status::OK(); + case ParquetColumnSchemaKind::STRUCT: { + DORIS_CHECK(!schema.children.empty()); + if (!format::is_partial_projection(projection)) { + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + DORIS_CHECK(!projection->children.empty()); + const auto child_id = projection->children.front().local_id(); + const auto child = std::ranges::find_if(schema.children, [child_id](const auto& candidate) { + return candidate->local_id == child_id; + }); + if (child == schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", schema.name); + } + return find_count_leaf(**child, &projection->children.front(), leaf); + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto& element = *schema.children.front(); + return find_count_leaf(element, format::find_child_projection(projection, element.local_id), + leaf); + } + case ParquetColumnSchemaKind::MAP: + // The key leaf owns entry existence and top-level MAP shape. Choosing it also avoids + // reading a potentially huge value BYTE_ARRAY for COUNT(map_col). + DORIS_CHECK(!schema.children.empty()); + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); +} + +NativeFieldSchema* find_physical_leaf(NativeFieldSchema* field, int physical_column_index) { + DORIS_CHECK(field != nullptr); + if (field->children.empty()) { + return field->physical_column_index == physical_column_index ? field : nullptr; + } + for (auto& child : field->children) { + if (auto* result = find_physical_leaf(&child, physical_column_index); result != nullptr) { + return result; + } + } + return nullptr; +} + +} // namespace + +CountColumnReader::CountColumnReader(std::string name, + std::unique_ptr level_reader, + ParquetColumnReaderProfile profile) + : _level_reader(std::move(level_reader)), _profile(profile), _name(std::move(name)) {} + +CountColumnReader::~CountColumnReader() { + sync_profile(); +} + +Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(reader != nullptr); + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + DORIS_CHECK(leaf_schema->leaf_column_id >= 0); + + const auto& thrift_metadata = metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::InvalidArgument("Invalid Parquet COUNT row group {}", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + if (leaf_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet COUNT leaf {} for row group {}", + leaf_schema->leaf_column_id, row_group_id); + } + + DORIS_CHECK(root_schema.local_id >= 0); + auto* root_field = + const_cast(metadata->schema().get_column(root_schema.local_id)); + DORIS_CHECK(root_field != nullptr); + auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); + if (leaf_field == nullptr) { + return Status::Corruption("Cannot resolve Parquet COUNT physical leaf {} for column {}", + leaf_schema->leaf_column_id, root_schema.name); + } + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + std::unique_ptr level_reader; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::LevelReader::create( + std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, + row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, + enable_page_cache, page_cache_file_key, compat, &level_reader)); + reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); + return Status::OK(); +} + +Status CountColumnReader::skip(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + { + SCOPED_TIMER(_profile.level_only_skip_time); + RETURN_IF_ERROR(_level_reader->skip_rows(static_cast(rows))); + } + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } + sync_profile(); + return Status::OK(); +} + +Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(rows_read != nullptr); + _definition_levels.clear(); + _repetition_levels.clear(); + _levels_written = 0; + size_t native_rows_read = 0; + { + SCOPED_TIMER(_profile.level_only_read_time); + RETURN_IF_ERROR(_level_reader->read_rows(static_cast(rows), &_repetition_levels, + &_definition_levels, &native_rows_read)); + } + *rows_read = static_cast(native_rows_read); + if (*rows_read != rows || _definition_levels.size() != _repetition_levels.size()) { + return Status::Corruption( + "Parquet COUNT level reader returned {} rows and {}/{} levels for {}", *rows_read, + _definition_levels.size(), _repetition_levels.size(), _name); + } + _levels_written = static_cast(_definition_levels.size()); + if (_levels_written < *rows_read) { + return Status::Corruption("Parquet COUNT returned {} levels for {} rows in column {}", + _levels_written, *rows_read, _name); + } + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); + } + sync_profile(); + return Status::OK(); +} + +void CountColumnReader::sync_profile() { + if (_level_reader == nullptr) { + return; + } + const auto stats = _level_reader->statistics(); + const auto& reported = _reported_native_stats; +#define UPDATE_NATIVE_PROFILE(counter, field) \ + if (_profile.counter != nullptr) { \ + COUNTER_UPDATE(_profile.counter, stats.field - reported.field); \ + } + UPDATE_NATIVE_PROFILE(decompress_time, decompress_time); + UPDATE_NATIVE_PROFILE(decompress_count, decompress_cnt); + UPDATE_NATIVE_PROFILE(decode_header_time, decode_header_time); + UPDATE_NATIVE_PROFILE(decode_value_time, decode_value_time); + UPDATE_NATIVE_PROFILE(decode_dictionary_time, decode_dict_time); + UPDATE_NATIVE_PROFILE(decode_level_time, decode_level_time); + UPDATE_NATIVE_PROFILE(skip_page_header_count, skip_page_header_num); + UPDATE_NATIVE_PROFILE(parse_page_header_count, parse_page_header_num); + UPDATE_NATIVE_PROFILE(read_page_header_time, read_page_header_time); + UPDATE_NATIVE_PROFILE(page_read_count, page_read_counter); + UPDATE_NATIVE_PROFILE(page_cache_write_count, page_cache_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_write_count, page_cache_compressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_write_count, + page_cache_decompressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_hit_count, page_cache_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_miss_count, page_cache_missing_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_hit_count, page_cache_compressed_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_hit_count, page_cache_decompressed_hit_counter); +#undef UPDATE_NATIVE_PROFILE + _reported_native_stats = stats; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h new file mode 100644 index 00000000000000..789da8534d2ce3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { +class NativeParquetMetadata; +struct ParquetColumnSchema; +// Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads +// are skipped in the encoding stream and never copied into Arrow builders or Doris strings. +class CountColumnReader { +public: + ~CountColumnReader(); + + static Status create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + Status skip(int64_t rows); + Status read_levels(int64_t rows, int64_t* rows_read); + + const std::vector& definition_levels() const { return _definition_levels; } + const std::vector& repetition_levels() const { return _repetition_levels; } + int64_t levels_written() const { return _levels_written; } + +private: + CountColumnReader(std::string name, std::unique_ptr level_reader, + ParquetColumnReaderProfile profile); + void sync_profile(); + + std::unique_ptr _level_reader; + ParquetColumnReaderProfile _profile; + std::string _name; + std::vector _definition_levels; + std::vector _repetition_levels; + native::ColumnChunkReaderStatistics _reported_native_stats; + int64_t _levels_written = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp deleted file mode 100644 index c042fc99b512aa..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/list_column_reader.h" - -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status ListColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status ListColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_batch(rows); -} - -Status ListColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_levels_batch(rows); -} - -Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", - _name); - } - DORIS_CHECK(_element_reader != nullptr); - ColumnArray* array_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr nested_column; - if (column != nullptr) { - array_column = array_column_from_output(*column); - DORIS_CHECK(array_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - nested_column = array_column->get_data_ptr()->assert_mutable(); - const auto& element_output_type = - assert_cast(*remove_nullable(_type)).get_nested_type(); - remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); - } - - const auto& def_levels = _element_reader->nested_definition_levels(); - const auto& rep_levels = _element_reader->nested_repetition_levels(); - const int64_t levels_written = _element_reader->nested_levels_written(); - std::vector entry_counts; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet LIST column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - int64_t child_value_count = 0; - if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(total_entries), nested_column, &child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(total_entries), &child_value_count)); - } - } else { - uint64_t pending_entries = 0; - auto flush_pending_entries = [&]() -> Status { - if (pending_entries == 0) { - return Status::OK(); - } - int64_t span_child_value_count = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(pending_entries), nested_column, - &span_child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(pending_entries), &span_child_value_count)); - } - if (span_child_value_count != static_cast(pending_entries)) { - return Status::Corruption( - "Parquet LIST column {} built {} child values, expected {}", _name, - span_child_value_count, pending_entries); - } - child_value_count += span_child_value_count; - pending_entries = 0; - return Status::OK(); - }; - - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - if (entry_count > 0) { - pending_entries += entry_count; - continue; - } - RETURN_IF_ERROR(flush_pending_entries()); - _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); - } - RETURN_IF_ERROR(flush_pending_entries()); - } - if (child_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", - _name, child_value_count, total_entries); - } - if (column != nullptr) { - array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& ListColumnReader::nested_definition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_definition_levels(); -} - -const std::vector& ListColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_repetition_levels(); -} - -int64_t ListColumnReader::nested_levels_written() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_levels_written(); -} - -bool ListColumnReader::is_or_has_repeated_child() const { - return true; -} - -void ListColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_element_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h deleted file mode 100644 index d64be546e394d7..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class ListColumnReader final : public ParquetColumnReader { -public: - ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr element_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _element_reader(std::move(element_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr - _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp deleted file mode 100644 index 8217d0c013abc0..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/map_column_reader.h" - -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status MapColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status MapColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); - return _value_reader->load_nested_batch(rows); -} - -Status MapColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); - return _value_reader->load_nested_levels_batch(rows); -} - -Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", - _name); - } - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ColumnMap* map_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr key_column; - MutableColumnPtr value_column; - if (column != nullptr) { - map_column = map_column_from_output(*column); - DORIS_CHECK(map_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - key_column = map_column->get_keys_ptr()->assert_mutable(); - value_column = map_column->get_values_ptr()->assert_mutable(); - const auto& map_output_type = assert_cast(*remove_nullable(_type)); - remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); - remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); - } - - const auto& def_levels = _key_reader->nested_definition_levels(); - const auto& rep_levels = _key_reader->nested_repetition_levels(); - const int64_t levels_written = _key_reader->nested_levels_written(); - - std::vector entry_counts; - std::vector map_level_indices; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - const int64_t current_level_idx = level_idx; - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - map_level_indices.push_back(current_level_idx); - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet MAP column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - int64_t key_value_count = 0; - size_t key_start = 0; - if (column != nullptr) { - key_start = key_column->size(); - RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), - key_column, &key_value_count)); - } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { - // MAP keys are required even if a projected Doris key type is nullable. Validate each - // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : map_level_indices) { - if (def_levels[key_level_idx] >= _definition_level) { - RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); - ++key_value_count; - } - } - scalar_key_reader->set_nested_build_level_cursor(level_idx); - } else { - RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), - &key_value_count)); - } - if (key_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, - key_value_count, total_entries); - } - if (column != nullptr) { - if (const auto* nullable_key_column = check_and_get_column(*key_column); - nullable_key_column != nullptr && - nullable_key_column->has_null(key_start, nullable_key_column->size())) { - return Status::Corruption("Parquet MAP column {} contains null key", _name); - } - } - int64_t value_count = 0; - if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { - const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); - const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); - const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); - int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : map_level_indices) { - while (value_level_idx < value_levels_written && - (value_rep_levels[value_level_idx] > _repetition_level || - value_def_levels[value_level_idx] < min_parent_definition_level || - (value_rep_levels[value_level_idx] >= _repetition_level && - value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { - ++value_level_idx; - } - if (value_level_idx >= value_levels_written) { - return Status::Corruption( - "Parquet MAP column {} value stream ended before key stream", _name); - } - // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, - // but the value stream still has one shape slot for every consumed MAP slot. Consume - // value slots in lockstep with key slots so shape-only slots from empty/null maps do - // not become scalar values. - if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { - return Status::Corruption( - "Parquet MAP column {} value repetition level is not aligned with key " - "stream", - _name); - } - if (def_levels[key_level_idx] >= _definition_level) { - if (column != nullptr) { - RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, - value_column)); - } else { - RETURN_IF_ERROR( - scalar_value_reader->validate_nested_value(value_level_idx, false)); - } - ++value_count; - } - ++value_level_idx; - } - scalar_value_reader->set_nested_build_level_cursor(value_level_idx); - } else { - // Complex MAP values own their nested shape below the entry slot, so they recursively - // process exactly one child value for each MAP entry. - if (column != nullptr) { - RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), - value_column, &value_count)); - } else { - RETURN_IF_ERROR(_value_reader->consume_nested_column( - static_cast(total_entries), &value_count)); - } - } - if (value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, - value_count, total_entries); - } - - if (column != nullptr) { - map_column->get_keys_ptr() = std::move(key_column); - map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& MapColumnReader::nested_definition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_definition_levels(); -} - -const std::vector& MapColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_repetition_levels(); -} - -int64_t MapColumnReader::nested_levels_written() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_levels_written(); -} - -bool MapColumnReader::is_or_has_repeated_child() const { - return true; -} - -void MapColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h deleted file mode 100644 index 1a8ca9c70d8c5b..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -// 2. build_nested_column() -> -class MapColumnReader final : public ParquetColumnReader { -public: - MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr key_reader, - std::unique_ptr value_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _key_reader(std::move(key_reader)), - _value_reader(std::move(value_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr _key_reader; // key column reader (always read fully) - std::unique_ptr - _value_reader; // value column reader (can be pruned by projection) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp new file mode 100644 index 00000000000000..3421ecdd296892 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" + +#include + +#include "util/hash_util.hpp" + +namespace doris::format::parquet::native { + +namespace { +Status set_hash_strategy(segment_v2::HashStrategyPB strategy, + std::function* hash_func) { + if (strategy != segment_v2::HashStrategyPB::XX_HASH_64) { + return Status::InvalidArgument("Invalid Parquet Bloom filter hash strategy {}", strategy); + } + *hash_func = [](const void* buf, int64_t len, uint64_t, void* out) { + *reinterpret_cast(out) = + HashUtil::xxhash64_compat_with_seed(reinterpret_cast(buf), len, 0); + }; + return Status::OK(); +} +} // namespace + +Status BlockSplitBloomFilter::init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) { + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _num_bytes = filter_size; + _size = _num_bytes; + _data = new char[_size]; + memset(_data, 0, _size); + _has_null = nullptr; + _is_write = true; + segment_v2::g_write_bloom_filter_num << 1; + segment_v2::g_write_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +Status BlockSplitBloomFilter::init(const char* buf, size_t size, + segment_v2::HashStrategyPB strategy) { + if (buf == nullptr || size <= 1) { + return Status::InvalidArgument("Invalid Parquet Bloom filter buffer of size {}", size); + } + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _data = new char[size]; + memcpy(_data, buf, size); + _size = size; + _num_bytes = size; + _has_null = nullptr; + segment_v2::g_read_bloom_filter_num << 1; + segment_v2::g_read_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +void BlockSplitBloomFilter::add_bytes(const char* buf, size_t size) { + DCHECK(buf != nullptr); + add_hash(hash(buf, size)); +} + +bool BlockSplitBloomFilter::test_bytes(const char* buf, size_t size) const { + return test_hash(hash(buf, size)); +} + +void BlockSplitBloomFilter::set_has_null(bool has_null) { + DCHECK(!has_null) << "Parquet Bloom filters do not track nulls"; +} + +void BlockSplitBloomFilter::set_masks(uint32_t key, BlockMask* block_mask) { + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + block_mask->item[i] = uint32_t {1} << ((key * SALT[i]) >> 27); + } +} + +void BlockSplitBloomFilter::add_hash(uint64_t hash) { + DCHECK_GE(_num_bytes, BYTES_PER_BLOCK); + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + bitset[bucket_index * BITS_SET_PER_BLOCK + i] |= block_mask.item[i]; + } +} + +bool BlockSplitBloomFilter::test_hash(uint64_t hash) const { + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + const auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + if ((bitset[bucket_index * BITS_SET_PER_BLOCK + i] & block_mask.item[i]) == 0) { + return false; + } + } + return true; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h new file mode 100644 index 00000000000000..38dc97712ac46c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/bloom_filter/bloom_filter.h" + +namespace doris::format::parquet::native { + +// Keep the split-block implementation inside v2; the native scan path must not link through the +// legacy Parquet reader merely to evaluate footer Bloom filters. +class BlockSplitBloomFilter final : public segment_v2::BloomFilter { +public: + Status init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) override; + Status init(const char* buf, size_t size, segment_v2::HashStrategyPB strategy) override; + void add_bytes(const char* buf, size_t size) override; + bool test_bytes(const char* buf, size_t size) const override; + void set_has_null(bool has_null) override; + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override; + bool test_hash(uint64_t hash) const override; + +private: + static constexpr int BYTES_PER_BLOCK = 32; + static constexpr int BITS_SET_PER_BLOCK = 8; + static constexpr uint32_t SALT[BITS_SET_PER_BLOCK] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + + struct BlockMask { + uint32_t item[BITS_SET_PER_BLOCK]; + }; + + static void set_masks(uint32_t key, BlockMask* block_mask); +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp new file mode 100644 index 00000000000000..06e4b239f3abb4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" + +#include + +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/bit_util.h" + +namespace doris::format::parquet::native { +Status BoolPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + std::array values; + size_t decoded = 0; + while (decoded < num_values) { + const size_t batch_size = std::min(values.size(), num_values - decoded); + for (size_t row = 0; row < batch_size; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain decoder"); + } + values[row] = static_cast(value); + } + RETURN_IF_ERROR(consumer.consume(values.data(), batch_size, sizeof(uint8_t))); + decoded += batch_size; + } + return Status::OK(); +} + +Status BoolPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + selected_values_.resize(selection.selected_values); + size_t range_index = 0; + size_t output = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + selected_values_[output++] = static_cast(value); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(selected_values_.data(), output, sizeof(uint8_t)); +} + +Status BoolPlainDecoder::skip_values(size_t num_values) { + int skip_cached = + std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); + unpacked_value_idx_ += skip_cached; + if (skip_cached == num_values) { + return Status::OK(); + } + int num_remaining = cast_set(num_values - skip_cached); + int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); + if (num_to_skip > 0) { + // A failed bulk skip must not be reported as success for a truncated boolean page. + if (!bool_values_.SkipBatch(1, num_to_skip)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + } + num_remaining -= num_to_skip; + if (num_remaining > 0) { + DCHECK_LE(num_remaining, UNPACKED_BUFFER_LEN); + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ < num_remaining)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + unpacked_value_idx_ = num_remaining; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h new file mode 100644 index 00000000000000..ee49a236b847ce --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { +/// Decoder bit-packed boolean-encoded values. +/// Implementation from https://github.com/apache/impala/blob/master/be/src/exec/parquet/parquet-bool-decoder.h +//bit-packed-run-len and rle-run-len must be in the range [1, 2^31 - 1]. +// This means that a Parquet implementation can always store the run length in a signed 32-bit integer +class BoolPlainDecoder final : public Decoder { +public: + BoolPlainDecoder() = default; + ~BoolPlainDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + bool_values_.Reset((const uint8_t*)data->data, data->size); + num_unpacked_values_ = 0; + unpacked_value_idx_ = 0; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&selected_values_, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return selected_values_.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return selected_values_.size() * sizeof(uint8_t); + } + +protected: + inline bool _decode_value(bool* value) { + if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { + *value = unpacked_values_[unpacked_value_idx_++]; + } else { + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ == 0)) { + return false; + } + *value = unpacked_values_[0]; + unpacked_value_idx_ = 1; + } + return true; + } + + /// A buffer to store unpacked values. Must be a multiple of 32 size to use the + /// batch-oriented interface of BatchedBitReader. We use uint8_t instead of bool because + /// bit unpacking is only supported for unsigned integers. The values are converted to + /// bool when returned to the user. + static const int UNPACKED_BUFFER_LEN = 128; + uint8_t unpacked_values_[UNPACKED_BUFFER_LEN]; + + /// The number of valid values in 'unpacked_values_'. + int num_unpacked_values_ = 0; + + /// The next value to return from 'unpacked_values_'. + int unpacked_value_idx_ = 0; + + /// Bit packed decoder, used if 'encoding_' is PLAIN. + BatchedBitReader bool_values_; + + std::vector selected_values_; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp new file mode 100644 index 00000000000000..d541550cef180f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" + +#include + +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +Status BoolRLEDecoder::set_data(Slice* slice) { + _data = slice; + _num_bytes = slice->size; + _offset = 0; + if (_num_bytes < 4) { + return Status::IOError("Received invalid length : " + std::to_string(_num_bytes) + + " (corrupt data page?)"); + } + // Load the first 4 bytes in little-endian, which indicates the length + const auto* data = reinterpret_cast(_data->data); + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > static_cast(_num_bytes - 4)) { + return Status::IOError("Received invalid number of bytes : " + std::to_string(num_bytes) + + " (corrupt data page?)"); + } + _num_bytes = num_bytes; + auto decoder_data = data + 4; + _decoder = RleBatchDecoder(const_cast(decoder_data), num_bytes, 1); + return Status::OK(); +} + +Status BoolRLEDecoder::skip_values(size_t num_values) { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), static_cast(batch_size)) != batch_size) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } + skipped += batch_size; + } + return Status::OK(); +} + +Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { + _values.resize(num_values); + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); + } + return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); +} + +Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + _values.resize(selection.total_values); + if (_decoder.GetBatch(_values.data(), cast_set(selection.total_values)) != + selection.total_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, + range.count * sizeof(uint8_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_values.data(), output, sizeof(uint8_t)); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h new file mode 100644 index 00000000000000..55d7a331bb27ca --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/rle_encoding.h" + +namespace doris { +class ColumnSelectVector; +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +class BoolRLEDecoder final : public Decoder { +public: + BoolRLEDecoder() = default; + ~BoolRLEDecoder() override = default; + + Status set_data(Slice* slice) override; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { return _values.size() * sizeof(uint8_t); } + +private: + RleBatchDecoder _decoder; + std::vector _values; + size_t _num_bytes; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp new file mode 100644 index 00000000000000..cc3dc66b3cacdc --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" + +#include "core/custom_allocator.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + _dict_items.clear(); + _dict = std::move(dict); + if (_dict == nullptr) { + return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); + } + if (UNLIKELY(length < 0)) { + return Status::Corruption("Wrong data length in dictionary"); + } + const size_t dict_length = cast_set(length); + // Every BYTE_ARRAY entry needs a four-byte length prefix; bound metadata-driven reserve first. + if (UNLIKELY(num_values > dict_length / sizeof(uint32_t))) { + return Status::Corruption("Dictionary value count exceeds byte array payload"); + } + _dict_items.reserve(num_values); + size_t offset_cursor = 0; + char* dict_item_address = reinterpret_cast(_dict.get()); + for (size_t i = 0; i < num_values; ++i) { + if (UNLIKELY(offset_cursor > dict_length || + dict_length - offset_cursor < sizeof(uint32_t))) { + return Status::Corruption("Wrong data length in dictionary"); + } + uint32_t l = decode_fixed32_le(_dict.get() + offset_cursor); + offset_cursor += sizeof(uint32_t); + if (UNLIKELY(l > dict_length - offset_cursor)) { + return Status::Corruption("Wrong data length in dictionary"); + } + _dict_items.emplace_back(dict_item_address + offset_cursor, l); + offset_cursor += l; + } + if (offset_cursor != dict_length) { + return Status::Corruption("Wrong dictionary data for byte array type"); + } + ++_dictionary_generation; + return Status::OK(); +} + +Status ByteArrayDictDecoder::decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return binary_consumer.consume(_dict_items.data(), _dict_items.size()); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h new file mode 100644 index 00000000000000..790d9f2e4b65ba --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/string_ref.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteArrayDictDecoder final : public BaseDictDecoder { +public: + ByteArrayDictDecoder() = default; + ~ByteArrayDictDecoder() override = default; + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override; + + size_t dictionary_size() const override { return _dict_items.size(); } + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override; + +protected: + // StringRef entries point into BaseDictDecoder::_dict. The dictionary buffer lives for the + // entire column chunk, so retaining a second copy of every byte is unnecessary. + DorisVector _dict_items; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp new file mode 100644 index 00000000000000..064bcffec8dd1f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" + +#include +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" + +namespace doris::format::parquet::native { +namespace { +Status read_length(const Slice* data, size_t* offset, uint32_t* length) { + if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { + return Status::IOError("Can't read byte array length from plain decoder"); + } + *length = decode_fixed32_le(reinterpret_cast(data->data) + *offset); + *offset += sizeof(uint32_t); + return Status::OK(); +} +} // namespace + +Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(num_values); + _value_offsets.clear(); + _value_offsets.reserve(num_values + 1); + _value_offsets.push_back(0); + for (size_t row = 0; row < num_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain decoder"); + } + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY batch exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + _offset += length; + } + _value_spans.clear(); + if (num_values != 0) { + _value_spans.push_back({.first = 0, .count = num_values}); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), num_values, _value_spans); +} + +Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(selection.selected_values); + _value_offsets.clear(); + _value_offsets.reserve(selection.selected_values + 1); + _value_offsets.push_back(0); + _value_spans.clear(); + _value_spans.reserve(selection.ranges.size()); + size_t range_index = 0; + size_t selected_in_range = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + if (selected_in_range != 0) { + _value_spans.push_back({.first = _payload_offsets.size() - selected_in_range, + .count = selected_in_range}); + selected_in_range = 0; + } + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY selection exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + ++selected_in_range; + } + _offset += length; + } + if (selected_in_range != 0) { + _value_spans.push_back( + {.first = _payload_offsets.size() - selected_in_range, .count = selected_in_range}); + } + DORIS_CHECK_EQ(_payload_offsets.size(), selection.selected_values); + if (_payload_offsets.empty()) { + return Status::OK(); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), _payload_offsets.size(), + _value_spans); +} + +Status ByteArrayPlainDecoder::skip_values(size_t num_values) { + for (int i = 0; i < num_values; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't skip enough bytes in plain decoder"); + } + _offset += length; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h new file mode 100644 index 00000000000000..45bc82fe823f1a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_util.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +template +class ColumnDecimal; +} // namespace doris + +namespace doris::format::parquet::native { +class ByteArrayPlainDecoder final : public Decoder { +public: + ByteArrayPlainDecoder() = default; + ~ByteArrayPlainDecoder() override = default; + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_payload_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_spans, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return (_payload_offsets.capacity() + _value_offsets.capacity()) * sizeof(uint32_t) + + _value_spans.capacity() * sizeof(ParquetSelectionRange); + } + size_t active_scratch_bytes() const override { + return (_payload_offsets.size() + _value_offsets.size()) * sizeof(uint32_t) + + _value_spans.size() * sizeof(ParquetSelectionRange); + } + +private: + std::vector _payload_offsets; + std::vector _value_offsets; + std::vector _value_spans; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp new file mode 100644 index 00000000000000..caf3b675e2dfd7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" + +#include +#include + +#include "core/column/column_fixed_length_object.h" +#include "util/byte_stream_split.h" + +namespace doris::format::parquet::native { +Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); + } + const int64_t stride = static_cast(_data->size / _type_length); + _decoded_values.resize(byte_size); + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + _offset / _type_length, num_values, stride, _decoded_values.data()); + _offset += byte_size; + return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); +} + +Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet byte-stream-split selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError( + "Out-of-bounds access in Parquet byte-stream-split selection decoder"); + } + DORIS_CHECK_EQ(_data->size % value_width, 0); + const int64_t stride = static_cast(_data->size / value_width); + _decoded_values.resize(selection.selected_values * value_width); + size_t output = 0; + const size_t first_row = _offset / value_width; + for (const auto& range : selection.ranges) { + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + first_row + range.first, range.count, stride, + _decoded_values.data() + output * value_width); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + _offset += input_bytes; + return consumer.consume(_decoded_values.data(), output, value_width); +} + +Status ByteStreamSplitDecoder::skip_values(size_t num_values) { + // Check in row units before multiplication so a corrupt skip count cannot wrap back in-bounds. + if (UNLIKELY(_type_length <= 0 || _offset > _data->size || + num_values > (_data->size - _offset) / _type_length)) { + return Status::IOError( + "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, + _data->size); + } + _offset += _type_length * num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h new file mode 100644 index 00000000000000..dfe1d954b61641 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteStreamSplitDecoder final : public Decoder { +public: + ByteStreamSplitDecoder() = default; + ~ByteStreamSplitDecoder() override = default; + + Status set_data(Slice* data) override { + if (UNLIKELY(_type_length <= 0 || data == nullptr || + data->size % static_cast(_type_length) != 0)) { + // Every byte lane must contain the same number of complete physical values. + return Status::Corruption("Invalid Parquet byte-stream-split page length"); + } + return Decoder::set_data(data); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_decoded_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _decoded_values.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _decoded_values.size() * sizeof(uint8_t); + } + +private: + std::vector _decoded_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp new file mode 100644 index 00000000000000..c219a7aef03be2 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -0,0 +1,1803 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_chunk_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "core/column/column.h" +#include "core/column/column_decimal.h" +#include "core/column/column_dictionary.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/bit_util.h" +#include "util/block_compression.h" +#include "util/cpu_info.h" +#include "util/unaligned.h" + +namespace cctz { +class time_zone; +} // namespace cctz +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { + +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available) { + return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; +} + +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed) { + const bool is_v2 = header.__isset.data_page_header_v2; + const bool page_is_compressed = + codec != tparquet::CompressionCodec::UNCOMPRESSED && + (!is_v2 || header.data_page_header_v2.is_compressed || data_page_v2_always_compressed); + if (!page_is_compressed && + UNLIKELY(header.compressed_page_size != header.uncompressed_page_size)) { + // An uncompressed payload has one physical representation, so accepting two lengths makes + // cold reads and decompressed cache hits consume different byte boundaries. + return Status::Corruption( + "Uncompressed Parquet page sizes differ: compressed={}, uncompressed={}", + header.compressed_page_size, header.uncompressed_page_size); + } + return Status::OK(); +} + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { + if (created_by.empty()) { + return {}; + } + const ::parquet::ApplicationVersion version(created_by); + return {.parquet_816_padding = + version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()), + .data_page_v2_always_compressed = version.VersionLt( + ::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())}; +} + +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range) { + DORIS_CHECK(range != nullptr); + int64_t start = metadata.data_page_offset; + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset > 0 && + metadata.dictionary_page_offset < start) { + // Some writers use dictionary_page_offset=0 as an absence sentinel. Range validation must + // follow has_dict_page() or the native reader starts at the Parquet magic bytes. + start = metadata.dictionary_page_offset; + } + const int64_t length = metadata.total_compressed_size; + if (UNLIKELY(start < 0 || length < 0)) { + return Status::Corruption("Parquet column chunk has a negative offset or length"); + } + const uint64_t unsigned_start = static_cast(start); + const uint64_t unsigned_length = static_cast(length); + if (UNLIKELY(unsigned_start > file_size || unsigned_length > file_size - unsigned_start)) { + // Thrift range fields are signed and untrusted; validate before converting them to the + // unsigned stream-reader coordinates so overflow cannot wrap back into the file. + return Status::Corruption("Parquet column chunk [{}, {}) exceeds file size {}", start, + unsigned_start + unsigned_length, file_size); + } + size_t bounded_length = static_cast(unsigned_length); + if (parquet_816_padding) { + // parquet-mr before PARQUET-816 under-reported the chunk by up to 100 bytes. Padding stays + // file-bounded and is only enabled for the affected writer versions. + bounded_length += std::min(100, file_size - unsigned_start - unsigned_length); + } + range->offset = static_cast(unsigned_start); + range->length = bounded_length; + return Status::OK(); +} + +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count) { + if (index.page_locations.empty() || data_page_offset < 0 || row_count < 0 || + index.page_locations.front().first_row_index != 0 || + index.page_locations.front().offset != data_page_offset || + chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { + return false; + } + // Row indexes alone cannot detect a uniformly shifted OffsetIndex. Anchor its first location + // to the owning metadata so page-to-row mapping cannot silently move by one physical page. + const uint64_t chunk_begin = chunk_range.offset; + const uint64_t chunk_end = chunk_begin + chunk_range.length; + uint64_t previous_end = chunk_begin; + int64_t previous_row = -1; + for (const auto& location : index.page_locations) { + if (location.first_row_index <= previous_row || location.first_row_index >= row_count || + location.offset < 0 || location.compressed_page_size <= 0) { + return false; + } + const uint64_t begin = static_cast(location.offset); + const uint64_t size = static_cast(location.compressed_page_size); + if (begin < chunk_begin || begin < previous_end || begin > chunk_end || + size > chunk_end - begin) { + return false; + } + previous_row = location.first_row_index; + previous_end = begin + size; + } + return true; +} + +namespace { + +class EmptyValueSectionDecoder final : public Decoder { +public: + Status skip_values(size_t num_values) override { + if (UNLIKELY(num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + return Status::OK(); + } +}; + +Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, + const ParquetInt96Timestamp& value, + const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; + static constexpr int64_t MICROS_PER_SECOND = 1000000LL; + + // Arrow normalized out-of-day INT96 nanos before the native V2 path replaced it. Preserve that + // legacy-writer compatibility here; rejecting the carrier loses valid pre-epoch/year-0 values + // already accepted by Doris external-table scans. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // Truncate the signed nanos field before day normalization. Flooring a negative value first + // changes the historical result by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); + int64_t epoch_seconds = micros / MICROS_PER_SECOND; + int64_t micros_of_second = micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + DateV2Value datetime; + datetime.from_unixtime(epoch_seconds, timezone); + datetime.set_microsecond(static_cast(micros_of_second)); + if (!datetime.is_valid_date()) { + return Status::DataQualityError("Parquet INT96 timestamp is outside the Doris range"); + } + data.push_back(datetime); + return Status::OK(); +} + +class V2Int96DateTimeConsumer final : public ParquetFixedValueConsumer { +public: + V2Int96DateTimeConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state) + : _data(assert_cast(column).get_data()), _state(state) { + static const auto utc = cctz::utc_time_zone(); + _timezone = context.timezone == nullptr ? &utc : context.timezone; + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load( + values + row * sizeof(ParquetInt96Timestamp)); + const auto status = append_v2_int96_datetime(_data, value, *_timezone); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + ParquetMaterializationState* _state; + const cctz::time_zone* _timezone = nullptr; +}; + +class RejectV2Int96BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("INT96 cannot be decoded from binary Parquet values"); + } +}; + +Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) { + V2Int96DateTimeConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + V2Int96DateTimeConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectV2Int96BinaryConsumer binary_consumer; + const auto dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + return state.materialize_dictionary(column); +} + +Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, + ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + auto* output = check_and_get_column(&column); + if (output == nullptr) { + return Status::InternalError("Dictionary indices require an INT32 output column"); + } + // Dictionary IDs have the same RLE/bit-packed representation for every physical type. + // Decode them before dispatching to a typed SerDe; otherwise a fixed-width SerDe treats + // the IDs as values and the row filter indexes its bitmap with materialized data. + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = output->size(); + auto& indices = output->get_data(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + if (context.physical_type == ParquetPhysicalType::INT96 && + check_and_get_column(&column) != nullptr) { + return read_v2_int96_datetime(column, source, context, num_values, state); + } + return serde.read_column_from_parquet(column, source, context, num_values, state); +} + +Status translate_value_encoding(tparquet::Encoding::type encoding, + ParquetValueEncoding* translated) { + DORIS_CHECK(translated != nullptr); + switch (encoding) { + case tparquet::Encoding::PLAIN: + *translated = ParquetValueEncoding::PLAIN; + return Status::OK(); + case tparquet::Encoding::RLE_DICTIONARY: + case tparquet::Encoding::PLAIN_DICTIONARY: + *translated = ParquetValueEncoding::DICTIONARY; + return Status::OK(); + case tparquet::Encoding::RLE: + *translated = ParquetValueEncoding::RLE; + return Status::OK(); + case tparquet::Encoding::BIT_PACKED: + *translated = ParquetValueEncoding::BIT_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + *translated = ParquetValueEncoding::DELTA_BINARY_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_LENGTH_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + *translated = ParquetValueEncoding::BYTE_STREAM_SPLIT; + return Status::OK(); + default: + return Status::NotSupported("Unsupported Parquet encoding {}", + tparquet::to_string(encoding)); + } +} + +template +Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decoder& decoder, + const ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector, + int64_t* materialization_time) { + SCOPED_RAW_TIMER(materialization_time); + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + RETURN_IF_ERROR( + read_native_or_serde(column, serde, decoder, context, run_length, state)); + break; + case ColumnSelectVector::NULL_DATA: + column.insert_many_defaults(run_length); + break; + case ColumnSelectVector::FILTERED_CONTENT: + RETURN_IF_ERROR(decoder.skip_values(run_length)); + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + return Status::OK(); +} + +// Presents one sparse page request as an ordinary sequential source to DataTypeSerDe. SerDe is +// entered once per page fragment; the concrete decoder decides whether to gather selected spans, +// batch-decode and compact, or use the cursor-preserving range fallback. +class SelectedDecodeSource final : public ParquetDecodeSource { +public: + SelectedDecodeSource(Decoder& decoder, const ParquetSelection& selection) + : _decoder(decoder), _selection(selection) {} + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_fixed_values(_selection, consumer); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_binary_values(_selection, consumer); + } + + Status skip_values(size_t num_values) override { + return Status::NotSupported("Selected Parquet source cannot be skipped, values={}", + num_values); + } + + bool has_dictionary() const override { return _decoder.has_dictionary(); } + uint64_t dictionary_generation() const override { return _decoder.dictionary_generation(); } + size_t dictionary_size() const override { return _decoder.dictionary_size(); } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return _decoder.decode_dictionary(fixed_consumer, binary_consumer); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_indices(_selection, indices); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_values(_selection, consumer); + } + + bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const override { + // Avoid touching a dictionary larger than L2 for rows already removed by sparse selection. + // Cache-resident or dense batches instead fuse RLE decode and gather, eliminating the + // page-sized intermediate ID vector. + return _selection.selected_values < _selection.total_values && + dictionary_bytes > static_cast(CpuInfo::get_l2_cache_size()); + } + +private: + Decoder& _decoder; + const ParquetSelection& _selection; +}; + +Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = select_vector.num_values(); + selection.selected_values = 0; + + size_t cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + DORIS_CHECK(read_type == ColumnSelectVector::CONTENT || + read_type == ColumnSelectVector::FILTERED_CONTENT); + if (read_type == ColumnSelectVector::CONTENT) { + selection.ranges.push_back({.first = cursor, .count = run_length}); + selection.selected_values += run_length; + } + cursor += run_length; + } + DORIS_CHECK_EQ(cursor, selection.total_values); + if (selection.selected_values == 0) { + return decoder.skip_values(selection.total_values); + } + + SCOPED_RAW_TIMER(materialization_time); + SelectedDecodeSource selected_source(decoder, selection); + return read_native_or_serde(column, serde, selected_source, context, selection.selected_values, + state); +} + +template +bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { +#define VISIT_COLUMN(TYPE) \ + if (auto* typed = check_and_get_column(&column)) { \ + visitor(*typed); \ + return true; \ + } + VISIT_COLUMN(ColumnUInt8) + VISIT_COLUMN(ColumnInt8) + VISIT_COLUMN(ColumnInt16) + VISIT_COLUMN(ColumnInt32) + VISIT_COLUMN(ColumnInt64) + VISIT_COLUMN(ColumnInt128) + VISIT_COLUMN(ColumnDate) + VISIT_COLUMN(ColumnDateTime) + VISIT_COLUMN(ColumnDateV2) + VISIT_COLUMN(ColumnDateTimeV2) + VISIT_COLUMN(ColumnFloat32) + VISIT_COLUMN(ColumnFloat64) + VISIT_COLUMN(ColumnIPv4) + VISIT_COLUMN(ColumnIPv6) + VISIT_COLUMN(ColumnTimeV2) + VISIT_COLUMN(ColumnTimeStampTz) + VISIT_COLUMN(ColumnOffset32) + VISIT_COLUMN(ColumnOffset64) + VISIT_COLUMN(ColumnDecimal32) + VISIT_COLUMN(ColumnDecimal64) + VISIT_COLUMN(ColumnDecimal128V2) + VISIT_COLUMN(ColumnDecimal128V3) + VISIT_COLUMN(ColumnDecimal256) + VISIT_COLUMN(ColumnString) + VISIT_COLUMN(ColumnString64) + VISIT_COLUMN(ColumnVarbinary) + VISIT_COLUMN(ColumnDictI32) +#undef VISIT_COLUMN + return false; +} + +template +void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + auto& data = column.get_data(); + DORIS_CHECK_EQ(data.size(), old_size + compact_values); + data.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + data[old_size + output] = typename ColumnType::value_type {}; + } else { + DORIS_CHECK(source > 0); + --source; + data[old_size + output] = std::move(data[old_size + source]); + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_string_values(ColumnStr& column, size_t old_size, + size_t compact_values, const NullMap& selected_nulls) { + auto& offsets = column.get_offsets(); + DORIS_CHECK_EQ(offsets.size(), old_size + compact_values); + const Offset prefix_end = old_size == 0 ? 0 : offsets[old_size - 1]; + offsets.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] == 0) { + DORIS_CHECK(source > 0); + --source; + offsets[old_size + output] = offsets[old_size + source]; + } else { + offsets[old_size + output] = source == 0 ? prefix_end : offsets[old_size + source - 1]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_pod_values(column, old_size, compact_values, selected_nulls); +} + +template +void expand_nullable_values(ColumnStr& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_string_values(column, old_size, compact_values, selected_nulls); +} + +void remap_nullable_conversion_failures(IColumn::Filter* conversion_failure_null_map, + size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + if (conversion_failure_null_map == nullptr) { + return; + } + DORIS_CHECK(conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + size_t source = compact_values; + // Walk backwards so writing an expanded row cannot overwrite an unread compact failure bit. + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + (*conversion_failure_null_map)[old_size + output] = 1; + } else { + DORIS_CHECK(source > 0); + --source; + (*conversion_failure_null_map)[old_size + output] = + (*conversion_failure_null_map)[old_size + source]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, NullMap& selected_nulls, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = 0; + selection.selected_values = 0; + selected_nulls.clear(); + selected_nulls.reserve(select_vector.num_values() - select_vector.num_filtered()); + + size_t physical_cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls.resize_fill(selected_nulls.size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls.resize_fill(selected_nulls.size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls.size(), + select_vector.num_values() - select_vector.num_filtered()); + + const size_t old_size = column.size(); + SCOPED_RAW_TIMER(materialization_time); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(decoder.skip_values(selection.total_values)); + column.insert_many_defaults(selected_nulls.size()); + return Status::OK(); + } + + if (state.conversion_failure_null_map != nullptr) { + DORIS_CHECK(state.conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + memset(state.conversion_failure_null_map->data() + old_size, 0, selection.selected_values); + } + SelectedDecodeSource selected_source(decoder, selection); + const auto status = read_native_or_serde(column, serde, selected_source, context, + selection.selected_values, state); + if (!status.ok()) { + if (state.conversion_failure_null_map != nullptr) { + memcpy(state.conversion_failure_null_map->data() + old_size, selected_nulls.data(), + selected_nulls.size()); + } + return status; + } + DORIS_CHECK_EQ(column.size(), old_size + selection.selected_values); + + remap_nullable_conversion_failures(state.conversion_failure_null_map, old_size, + selection.selected_values, selected_nulls); + const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { + // Decode into the final nested column compactly, then expand in place. This preserves the + // V2 no-intermediate-column invariant while restoring the nullable sparse row layout. + expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); + }); + DORIS_CHECK(expanded); + return Status::OK(); +} + +class FixedWidthPredicateConsumer final : public ParquetFixedValueConsumer { +public: + FixedWidthPredicateConsumer(const VExprSPtrs& conjuncts, DataTypePtr data_type, int column_id, + IColumn::Filter* matches, IColumn* projected_column) + : _conjuncts(conjuncts), + _data_type(std::move(data_type)), + _column_id(column_id), + _matches(matches), + _projected_column(projected_column) { + DORIS_CHECK(_matches != nullptr); + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _matches->size(); + _matches->resize_fill(old_size + num_values, 1); + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->execute_on_raw_fixed_values(values, num_values, value_width, + _data_type, _column_id, + _matches->data() + old_size)); + } + if (_projected_column != nullptr) { + size_t row = 0; + while (row < num_values) { + while (row < num_values && (*_matches)[old_size + row] == 0) { + ++row; + } + const size_t run_begin = row; + while (row < num_values && (*_matches)[old_size + row] != 0) { + ++row; + } + if (row != run_begin) { + _projected_column->insert_many_raw_data( + reinterpret_cast(values + run_begin * value_width), + row - run_begin); + } + } + } + return Status::OK(); + } + +private: + const VExprSPtrs& _conjuncts; + DataTypePtr _data_type; + int _column_id; + IColumn::Filter* _matches; + IColumn* _projected_column; +}; + +} // namespace + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range) + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), + _stream_reader(reader), + _metadata(column_chunk->meta_data), + _offset_index(offset_index), + _total_rows(total_rows), + _io_ctx(io_ctx), + _page_read_ctx(page_read_ctx) { + if (chunk_range != nullptr) { + _chunk_range = *chunk_range; + _has_validated_chunk_range = true; + } +} + +template +Status ColumnChunkReader::init() { + size_t start_offset = _has_validated_chunk_range + ? _chunk_range.offset + : (has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset); + size_t chunk_size = + _has_validated_chunk_range ? _chunk_range.length : _metadata.total_compressed_size; + // create page reader + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, + _page_read_ctx, _offset_index); + // get the block compression codec + RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); + _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); + return Status::OK(); +} + +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels, size_t start_index) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + DORIS_CHECK(start_index <= def_levels.size()); + for (size_t idx = start_index; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } + } + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::read_levels( + size_t num_values, std::vector* rep_levels, std::vector* def_levels) { + DORIS_CHECK(rep_levels != nullptr); + DORIS_CHECK(def_levels != nullptr); + if (_remaining_num_values < num_values || _remaining_rep_nums < num_values || + _remaining_def_nums < num_values) { + return Status::Corruption( + "Parquet level reader requested {} slots with only {}/{}/{} remaining", num_values, + _remaining_num_values, _remaining_rep_nums, _remaining_def_nums); + } + + const size_t start_index = def_levels->size(); + rep_levels->resize(rep_levels->size() + num_values, 0); + def_levels->resize(def_levels->size() + num_values, 0); + if (_max_rep_level > 0) { + const size_t decoded = _rep_level_decoder.get_levels( + rep_levels->data() + rep_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet repetition level stream ended after {} of {} slots", + decoded, num_values); + } + } + if (_max_def_level > 0) { + const size_t decoded = _def_level_decoder.get_levels( + def_levels->data() + def_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet definition level stream ended after {} of {} slots", + decoded, num_values); + } + } + _remaining_rep_nums -= num_values; + _remaining_def_nums -= num_values; + return skip_nested_values(*def_levels, start_index); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + _state = INITIALIZED; + return parse_page_header(); + } + if (header->type != tparquet::PageType::DICTIONARY_PAGE) { + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + _state = INITIALIZED; + continue; + } + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; + // A dictionary is the only non-data page with decoder state. Any following index or + // extension pages are skipped by the same pre-data loop. + } +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); + } + const tparquet::PageHeader* header = nullptr; + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + break; + } + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + return Status::Corruption("Parquet dictionary page appears after data pages"); + } + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + } + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + if constexpr (IN_COLLECTION && OFFSET_INDEX) { + if (!_page_reader->is_header_v2() && _page_reader->has_active_offset_index()) { + // V1 nested pages do not declare their logical row count. An OffsetIndex span cannot + // be trusted until repetition levels are decoded, so keep the sequential cursor path. + _page_reader->discard_offset_index(); + _offset_index = nullptr; + } + } + const bool active_offset_index = _page_reader->has_active_offset_index(); + if (page_num_values < 0 || page_num_values > _metadata.num_values || + (!active_offset_index && + static_cast(page_num_values) > + static_cast(_metadata.num_values) - _chunk_parsed_values)) { + // Page counts are untrusted and feed both level decoders and scratch sizing. Bound each + // page by the column metadata before converting to unsigned counters. + return Status::Corruption("Parquet data page value count {} exceeds column total {}", + page_num_values, _metadata.num_values); + } + if constexpr (!IN_COLLECTION) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(page_num_values) != page_end_row - page_start_row)) { + // Flat columns have exactly one physical value slot per logical row. Rejecting a + // divergent header/OffsetIndex span prevents every later page from shifting rows. + return Status::Corruption( + "Parquet flat data page has {} values for logical row range [{}, {})", + page_num_values, page_start_row, page_end_row); + } + } else if (_page_reader->is_header_v2() && active_offset_index) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(header->data_page_header_v2.num_rows) != + page_end_row - page_start_row)) { + // V2 is the only repeated-page format that states its logical row count in the page + // header, so it must agree with the optional index before indexed seeking is allowed. + return Status::Corruption( + "Parquet nested data page has {} rows for indexed row range [{}, {})", + header->data_page_header_v2.num_rows, page_start_row, page_end_row); + } + } + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if (!active_offset_index) { + _chunk_parsed_values += _remaining_num_values; + } + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status ColumnChunkReader::next_page() { + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); + return Status::OK(); +} + +template +Status ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { + const size_t rl = page_v2.repetition_levels_byte_length; + const size_t dl = page_v2.definition_levels_byte_length; + if (UNLIKELY(rl > page_data.size || dl > page_data.size - rl)) { + // Validate the physical slice again because a cached entry may itself be truncated. + return Status::Corruption("Parquet data page v2 level bytes exceed available payload"); + } + _v2_rep_levels = Slice(page_data.data, rl); + _v2_def_levels = Slice(page_data.data + rl, dl); + page_data.data += dl + rl; + page_data.size -= dl + rl; + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::Corruption("Should parse page header"); + } + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + int32_t uncompressed_size = header->uncompressed_page_size; + bool page_loaded = false; + + // First, try to reuse a cache handle previously discovered by PageReader + // (header-only lookup) to avoid a second lookup here. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + size_t levels_size = 0; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + levels_size = rl + dl; + if (UNLIKELY(header_size > cached.size || + levels_size > cached.size - header_size)) { + return Status::Corruption("Cached Parquet page is shorter than its v2 levels"); + } + _v2_rep_levels = + Slice(reinterpret_cast(cached.data) + header_size, rl); + _v2_def_levels = + Slice(reinterpret_cast(cached.data) + header_size + rl, dl); + } + // payload_slice points to the bytes after header and levels + if (UNLIKELY(header_size + levels_size > cached.size)) { + return Status::Corruption("Cached Parquet page is shorter than its header"); + } + Slice payload_slice(cached.data + header_size + levels_size, + cached.size - header_size - levels_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + const size_t expected_payload_size = + cache_payload_is_decompressed + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->compressed_page_size) - levels_size; + if (UNLIKELY(payload_slice.size != expected_payload_size)) { + return Status::Corruption("Cached Parquet page payload has size {}, expected {}", + payload_slice.size, expected_payload_size); + } + + if (cache_payload_is_decompressed) { + // Cached payload is already uncompressed + _page_data = payload_slice; + } else { + CHECK(_block_compress_codec); + // Decompress cached payload into _decompress_buf for decoding + size_t uncompressed_payload_size = + header->__isset.data_page_header_v2 + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->uncompressed_page_size); + _reserve_decompress_buf(uncompressed_payload_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + if (UNLIKELY(_page_data.size != uncompressed_payload_size)) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_payload_size); + } + } + // page cache counters were incremented when PageReader did the header-only + // cache lookup. Do not increment again to avoid double-counting. + page_loaded = true; + } + } + + if (!page_loaded) { + const bool prepare_cache_payload = can_prepare_page_cache_payload( + _page_read_ctx.enable_parquet_file_page_cache, config::disable_storage_page_cache, + StoragePageCache::instance() != nullptr, !_page_reader->header_bytes().empty()); + if (_block_compress_codec != nullptr) { + Slice compressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + std::vector level_bytes; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + // uncompressed_size = rl + dl + uncompressed_data_size + // compressed_size = rl + dl + compressed_data_size + uncompressed_size -= header_v2.repetition_levels_byte_length + + header_v2.definition_levels_byte_length; + // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), compressed_data.data, level_sz); + } + // now remove levels from compressed_data for decompression + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); + } + bool is_v2_compressed = header->__isset.data_page_header_v2 && + (header->data_page_header_v2.is_compressed || + _page_read_ctx.data_page_v2_always_compressed); + bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; + + if (page_has_compression) { + // Decompress payload for immediate decoding + _reserve_decompress_buf(uncompressed_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + if (UNLIKELY(_page_data.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_size); + } + + // Decide whether to cache decompressed payload or compressed payload based on threshold + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (prepare_cache_payload) { + if (cache_payload_decompressed) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + // cache the compressed payload as-is (header | levels | compressed_payload) + _insert_page_into_cache( + level_bytes, Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + } else { + // no compression on this page, use the data directly + _page_data = Slice(compressed_data.data, compressed_data.size); + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } else { + // For uncompressed page, we may still need to extract v2 levels + std::vector level_bytes; + Slice uncompressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data)); + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), uncompressed_data.data, level_sz); + } + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, uncompressed_data)); + } + // copy page data out + _page_data = Slice(uncompressed_data.data, uncompressed_data.size); + // Optionally cache uncompressed data for uncompressed pages + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Initialize repetition level and definition level. Skip when level = 0, which means required field. + if (_max_rep_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, + _remaining_rep_nums)); + } else { + RETURN_IF_ERROR(_rep_level_decoder.init( + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, + _remaining_rep_nums)); + } + } + if (_max_def_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, + _remaining_def_nums)); + } else { + RETURN_IF_ERROR(_def_level_decoder.init( + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, + _remaining_def_nums)); + } + } + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; + // change the deprecated encoding to RLE_DICTIONARY + if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { + encoding = tparquet::Encoding::RLE_DICTIONARY; + } + _current_encoding = encoding; + + // Reuse page decoder + Decoder* encoding_decoder = nullptr; + if (_decoders.find(static_cast(encoding)) != _decoders.end()) { + encoding_decoder = _decoders[static_cast(encoding)].get(); + } else { + std::unique_ptr page_decoder; + RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + _decoders[static_cast(encoding)] = std::move(page_decoder); + encoding_decoder = _decoders[static_cast(encoding)].get(); + } + _empty_value_section = _page_data.empty() && _max_def_level > 0; + if (_empty_value_section) { + // Nullable all-NULL pages legally contain only definition levels. Keep them decodable for + // every advertised encoding, but make a non-NULL definition level fail before stale decoder + // state from the preceding page can be consumed. + if (_empty_value_decoder == nullptr) { + _empty_value_decoder = std::make_unique(); + } + _page_decoder = _empty_value_decoder.get(); + } else { + _page_decoder = encoding_decoder; + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); + } + + _state = DATA_LOADED; + return Status::OK(); +} + +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); + SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time); + + // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. + // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. + // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; + if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && + dict_encoding != tparquet::Encoding::PLAIN) { + return Status::InternalError("Unsupported dictionary encoding {}", + tparquet::to_string(dict_encoding)); + } + + // Prepare dictionary data + int32_t uncompressed_size = header->uncompressed_page_size; + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + auto dict_data = make_unique_buffer(uncompressed_size); + bool dict_loaded = false; + + // Try to load dictionary page from cache + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + // Dictionary page layout in cache: header | payload (compressed or uncompressed) + Slice payload_slice(cached.data + header_size, cached.size - header_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Use cached decompressed dictionary data + if (UNLIKELY(payload_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Cached Parquet dictionary payload has size {}, expected {}", + payload_slice.size, uncompressed_size); + } + memcpy(dict_data.get(), payload_slice.data, payload_slice.size); + dict_loaded = true; + } else { + CHECK(_block_compress_codec); + // Decompress cached compressed dictionary data + Slice dict_slice(dict_data.get(), uncompressed_size); + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + dict_loaded = true; + } + + // When dictionary page is loaded from cache, we need to skip the page data + // to update the offset correctly (similar to calling get_page_data()) + if (dict_loaded) { + _page_reader->skip_page_data(); + } + } + } + + if (!dict_loaded) { + // Load and decompress dictionary page from file + if (_block_compress_codec != nullptr) { + auto dict_num = header->dictionary_page_header.num_values; + if (dict_num == 0 && uncompressed_size != 0) { + return Status::IOError( + "Dictionary page's num_values is {} but uncompressed_size is {}", dict_num, + uncompressed_size); + } + Slice compressed_data; + Slice dict_slice(dict_data.get(), uncompressed_size); + if (dict_num != 0) { + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + // Dictionary probes stop before data pages, so count their decompression here or + // metadata pruning profiles will report no codec work for the scan. + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR( + _block_compress_codec->decompress(compressed_data, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + } + + // Decide whether to cache decompressed or compressed dictionary based on threshold + // If uncompressed_page_size == 0, should_cache_decompressed will return true + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; // Dictionary pages don't have levels + if (cache_payload_decompressed) { + // Cache the decompressed dictionary page + // If dict_num == 0, `dict_slice` will be empty + _insert_page_into_cache(empty_levels, dict_slice); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + DCHECK(!compressed_data.empty()); + // Cache the compressed dictionary page + _insert_page_into_cache(empty_levels, + Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + // `get_page_data` not called, we should skip the page data + // Because `_insert_page_into_cache` will use _page_reader, we should exec `skip_page_data` after `_insert_page_into_cache` + if (dict_num == 0) { + _page_reader->skip_page_data(); + } + } else { + Slice dict_slice; + RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + // The data is stored by BufferedStreamReader, we should copy it out + memcpy(dict_data.get(), dict_slice.data, dict_slice.size); + + // Cache the uncompressed dictionary page + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; + Slice payload(dict_data.get(), uncompressed_size); + _insert_page_into_cache(empty_levels, payload); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Cache page decoder + std::unique_ptr page_decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + // Set the dictionary data + RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, + header->dictionary_page_header.num_values)); + _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); + + _has_dict = true; + return Status::OK(); +} + +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { + if (size > _decompress_buf_size) { + _decompress_buf_size = BitUtil::next_power_of_two(size); + _decompress_buf = make_unique_buffer(_decompress_buf_size); + } +} + +template +void ColumnChunkReader::_insert_page_into_cache( + const std::vector& level_bytes, const Slice& payload) { + StoragePageCache::CacheKey key = + _page_reader->make_page_cache_key(_page_reader->header_start_offset()); + const std::vector& header_bytes = _page_reader->header_bytes(); + size_t total = header_bytes.size() + level_bytes.size() + payload.size; + auto page = std::make_unique(total, true, segment_v2::DATA_PAGE); + size_t pos = 0; + memcpy(page->data() + pos, header_bytes.data(), header_bytes.size()); + pos += header_bytes.size(); + if (!level_bytes.empty()) { + memcpy(page->data() + pos, level_bytes.data(), level_bytes.size()); + pos += level_bytes.size(); + } + if (payload.size > 0) { + memcpy(page->data() + pos, payload.data, payload.size); + pos += payload.size; + } + page->reset_size(total); + PageCacheHandle handle; + StoragePageCache::instance()->insert(key, page.get(), &handle, segment_v2::DATA_PAGE); + page.release(); + _chunk_statistics.page_cache_write_counter += 1; +} + +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { + if (UNLIKELY(_remaining_num_values < num_values)) { + return Status::IOError("Skip too many values in current page. {} vs. {}", + _remaining_num_values, num_values); + } + if (skip_data) { + if (UNLIKELY(_empty_value_section && num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); + } + // Commit logical page progress only after the physical decoder accepted the whole request. + _remaining_num_values -= num_values; + return Status::OK(); +} + +template +Status ColumnChunkReader::materialize_values( + MutableColumnPtr& doris_column, const DataTypeSerDe& serde, ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector) { + if (select_vector.num_values() == 0) { + return Status::OK(); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + const size_t physical_values = select_vector.num_values() - select_vector.num_nulls(); + if (UNLIKELY(_empty_value_section && physical_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + physical_values); + } + if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && + !_has_dict && physical_values != 0)) { + return Status::IOError("Not dictionary coded"); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + Status status; + if (select_vector.has_filter()) { + if (select_vector.num_nulls() == 0) { + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_non_null_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else if (visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + // Parquet omits NULL leaf values from the physical stream. For example, the logical + // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The + // generic fallback follows the logical selection runs, so those NULLs split one + // contiguous physical span into decode(d0), insert-NULL, decode(d1), insert-NULL, + // decode(d2). On nullable sparse scans this repeatedly enters SerDe and turns decimal + // scaling, date conversion, timestamp/timezone conversion, or dictionary-ID + // materialization into many tiny calls even though the physical values are adjacent. + // + // Build selected non-NULL ranges in physical coordinates instead. In the example this + // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as + // the logical NULL layout, and expands the final nested column backwards so unread + // compact values cannot be overwritten. Apply this to every expandable V2 scalar, + // including ordinary PLAIN primitives: otherwise nullable numeric predicates still + // fragment a physical span into millions of SerDe calls and defeat sparse decoding. + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_nullable_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + _nullable_selection_nulls, &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else { + ++_chunk_statistics.hybrid_selection_null_fallback_batches; + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + } + } else { + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, + &_chunk_statistics.materialization_time); + } + RETURN_IF_ERROR(status); + _remaining_num_values -= select_vector.num_values(); + return Status::OK(); +} + +template +bool ColumnChunkReader::can_filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id) const { + if (conjuncts.empty() || + !supports_raw_fixed_filter_encoding(_current_encoding, _metadata.type)) { + return false; + } + const auto primitive_type = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool has_identity_width = + (_metadata.type == tparquet::Type::INT32 && primitive_type == TYPE_INT) || + (_metadata.type == tparquet::Type::INT64 && primitive_type == TYPE_BIGINT) || + (_metadata.type == tparquet::Type::FLOAT && primitive_type == TYPE_FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && primitive_type == TYPE_DOUBLE); + if (!has_identity_width) { + // Raw predicates consume the physical Parquet width. Logical conversions such as UINT32 + // to BIGINT must stay on the typed path or a four-byte value is interpreted as eight bytes. + return false; + } + return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + }); +} + +template +Status ColumnChunkReader::filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, + NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(selected_nulls != nullptr); + DORIS_CHECK(physical_matches != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + *used_filter = false; + row_filter->clear(); + if (!can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + + ParquetSelection selection; + selected_nulls->clear(); + selected_nulls->reserve(select_vector.num_values() - select_vector.num_filtered()); + size_t physical_cursor = 0; + auto build_selection = [&]() { + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == + physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls->resize_fill(selected_nulls->size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls->resize_fill(selected_nulls->size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + }; + if (select_vector.has_filter()) { + build_selection.template operator()(); + } else { + build_selection.template operator()(); + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls->size(), + select_vector.num_values() - select_vector.num_filtered()); + if (UNLIKELY(_empty_value_section && selection.total_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + selection.total_values); + } + + physical_matches->clear(); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); + } else { + FixedWidthPredicateConsumer consumer(conjuncts, _field_schema->data_type, column_id, + physical_matches, projected_column); + RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); + DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); + } + + row_filter->reserve(selected_nulls->size()); + size_t physical_row = 0; + for (const uint8_t is_null : *selected_nulls) { + row_filter->push_back(is_null != 0 ? 0 : (*physical_matches)[physical_row++]); + } + DORIS_CHECK_EQ(physical_row, physical_matches->size()); + // Commit logical progress only after both the raw comparison and NULL remapping succeed. + _remaining_num_values -= select_vector.num_values(); + *used_filter = true; + return Status::OK(); +} + +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + return Status::OK(); + } + } + + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2() || !IN_COLLECTION) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, &result_rows, + &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + } + }; + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + // Reserve only the requested row frontier. One nested row may legitimately contain more + // values and grow the vector incrementally, but a forged page count must not allocate gigabytes + // before the level stream proves those values exist. + const size_t requested_frontier = + max_rows == std::numeric_limits::max() ? max_rows : max_rows + 1; + rep_levels.reserve(rep_levels.size() + + std::min(_remaining_rep_nums, requested_frontier)); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + // A continuation level is valid across later V1 pages only after this chunk has seen + // a row start; accepting it on the first sequential page invents an orphan parent row. + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + if ((_page_reader->is_header_v2() || _page_reader->has_active_offset_index()) && + UNLIKELY(_current_row != _page_reader->end_row())) { + // V2 and OffsetIndex advertise an exact logical row span. A page that exhausts its + // repetition levels without that many row starts would otherwise make the caller retry + // the same row forever. + return Status::Corruption( + "Parquet nested data page ended at row {}, expected page end row {}", _current_row, + _page_reader->end_row()); + } + + auto need_check_cross_page = [&]() -> bool { + return IN_COLLECTION && !_page_reader->has_active_offset_index() && + _remaining_rep_nums == 0 && !_page_reader->is_header_v2() && has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { + switch (_field_schema->physical_type) { + case tparquet::Type::INT32: + [[fallthrough]]; + case tparquet::Type::FLOAT: + return 4; + case tparquet::Type::INT64: + [[fallthrough]]; + case tparquet::Type::DOUBLE: + return 8; + case tparquet::Type::INT96: + return 12; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + return _field_schema->parquet_schema.type_length; + default: + return -1; + } +} + +/** + * Checks if the given column has a dictionary page. + * + * This function determines the presence of a dictionary page by checking the + * dictionary_page_offset field in the column metadata. The dictionary_page_offset + * must be set and greater than 0, and it must be less than the data_page_offset. + * + * The reason for these checks is based on the implementation in the Java version + * of ORC, where dictionary_page_offset is used to indicate the absence of a dictionary. + * Additionally, Parquet may write an empty row group, in which case the dictionary page + * content would be empty, and thus the dictionary page should not be read. + * + * See https://github.com/apache/arrow/pull/2667/files + */ +bool has_dict_page(const tparquet::ColumnMetaData& column) { + return column.__isset.dictionary_page_offset && column.dictionary_page_offset > 0 && + column.dictionary_page_offset < column.data_page_offset; +} + +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h new file mode 100644 index 00000000000000..5acf544bcc95da --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -0,0 +1,388 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "util/slice.h" + +namespace doris { +class BlockCompressionCodec; +class DataTypeSerDe; +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris::format::parquet::native { +using ::doris::ColumnString; + +struct ColumnChunkRange { + size_t offset = 0; + size_t length = 0; +}; + +struct ParquetReaderCompat { + bool parquet_816_padding = false; + bool data_page_v2_always_compressed = false; +}; + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by); +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range); +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count); +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available); +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed); + +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t materialization_time = 0; + int64_t hybrid_selection_batches = 0; + int64_t hybrid_selection_ranges = 0; + int64_t hybrid_selection_null_fallback_batches = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; +}; + +/** + * Read and decode parquet column data into doris block column. + *

Usage:

+ * // Create chunk reader + * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, + * tparquet::ColumnChunk* column_chunk, + * NativeFieldSchema* fieldSchema); + * // Initialize chunk reader + * chunk_reader.init(); + * while (chunk_reader.has_next_page()) { + * // Seek to next page header. Only read and parse the page header, not page data. + * chunk_reader.next_page(); + * // Load data to decoder. Load the page data into underlying container. + * // Or, we can call the chunk_reader.skip_page() to skip current page. + * chunk_reader.load_page_data(); + * // Decode values into column or slice. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. + * chunk_reader.materialize_values(column, serde, context, state, selection); + * } + */ +template +class ColumnChunkReader { +public: + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_row, io::IOContext* io_ctx, + const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range = nullptr); + ~ColumnChunkReader() = default; + + // Initialize chunk reader, will generate the decoder and codec. + Status init(); + + // Whether the chunk reader has a more page to read. + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + return _page_reader->has_next_page(); + } + } + // A nested V1 page can invalidate its offset index after initialization. In that mode the + // validated byte range may contain writer padding, so logical values—not remaining bytes— + // are the authoritative chunk boundary. + return _chunk_parsed_values < _metadata.num_values; + } + + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. + // when skip_data = false, the underlying decoder will not skip data, + // only used when maintaining the consistency of _remaining_num_values. + Status skip_values(size_t num_values, bool skip_data = true); + + // Load page data into the underlying container, + // and initialize the repetition and definition level decoder for current page data. + Status load_page_data(); + Status load_page_data_idempotent() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + return load_page_data(); + } + // The remaining number of values in current page(including null values). Decreased when reading or skipping. + uint32_t remaining_num_values() const { return _remaining_num_values; } + + // Apply one logical selection to the encoded page. Decoder advances physical payload cursors; + // SerDe interprets each selected raw span and materializes the destination Doris column. + Status materialize_values(MutableColumnPtr& doris_column, const DataTypeSerDe& serde, + ParquetDecodeContext& context, ParquetMaterializationState& state, + ColumnSelectVector& select_vector); + + static bool supports_raw_fixed_filter_encoding(tparquet::Encoding::type encoding, + tparquet::Type::type physical_type) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::DELTA_BINARY_PACKED: + return physical_type == tparquet::Type::INT32 || physical_type == tparquet::Type::INT64; + default: + return false; + } + } + + // Evaluate selected fixed-width values and return one keep byte per selected logical row. + // NULL comparisons are false and therefore never enter the physical consumer; non-null + // matches are appended to projected_column when requested. + Status filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id, + ColumnSelectVector& select_vector, NullMap* selected_nulls, + IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + bool can_filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id) const; + + // Get the repetition level decoder of current page. + LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } + // Get the definition level decoder of current page. + LevelDecoder& def_level_decoder() { return _def_level_decoder; } + + level_t max_rep_level() const { return _max_rep_level; } + level_t max_def_level() const { return _max_def_level; } + + bool has_dict() const { return _has_dict; }; + + // Get page decoder + Decoder* get_page_decoder() { return _page_decoder; } + + void release_decoder_scratch(size_t max_retained_bytes) { + for (auto& [encoding, decoder] : _decoders) { + decoder->release_scratch(max_retained_bytes); + } + // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. + _rep_level_decoder.release_scratch(max_retained_bytes); + _def_level_decoder.release_scratch(max_retained_bytes); + } + + size_t retained_decoder_scratch_bytes() const { + size_t bytes = _rep_level_decoder.retained_scratch_bytes() + + _def_level_decoder.retained_scratch_bytes(); + for (const auto& [encoding, decoder] : _decoders) { + bytes += decoder->retained_scratch_bytes(); + } + return bytes; + } + + size_t active_decoder_scratch_bytes() const { + // Only the current encoding is active. Old decoder instances retain reusable capacity but + // must not make the high-water policy treat their last batch as current working memory. + return (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + _rep_level_decoder.active_scratch_bytes() + + _def_level_decoder.active_scratch_bytes(); + } + + tparquet::Encoding::type current_encoding() const { return _current_encoding; } + + ColumnChunkReaderStatistics& chunk_statistics() { + _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time; + _chunk_statistics.skip_page_header_num = + _page_reader->page_statistics().skip_page_header_num; + _chunk_statistics.parse_page_header_num = + _page_reader->page_statistics().parse_page_header_num; + _chunk_statistics.read_page_header_time = + _page_reader->page_statistics().read_page_header_time; + // PageReader statistics are already cumulative. Snapshot them instead of folding the same + // totals into the chunk on every FileScannerV2 batch. Cache write counters are owned by + // ColumnChunkReader because insertion happens after decompression and therefore remain in + // the chunk accumulator above. + _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.data_page_read_counter = + _page_reader->page_statistics().data_page_read_counter; + _chunk_statistics.page_cache_hit_counter = + _page_reader->page_statistics().page_cache_hit_counter; + _chunk_statistics.page_cache_missing_counter = + _page_reader->page_statistics().page_cache_missing_counter; + _chunk_statistics.page_cache_compressed_hit_counter = + _page_reader->page_statistics().page_cache_compressed_hit_counter; + _chunk_statistics.page_cache_decompressed_hit_counter = + _page_reader->page_statistics().page_cache_decompressed_hit_counter; + return _chunk_statistics; + } + + Decoder* dictionary_decoder() { + return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)].get(); + } + + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + // Decode level slots without materializing their values. `read_levels` is the flat-column + // counterpart of load_page_nested_rows()+fill_def(): both advance definition/repetition and + // value streams together so a levels-only consumer cannot desynchronize the next data page. + Status read_levels(size_t num_values, std::vector* rep_levels, + std::vector* def_levels); + Status skip_nested_values(const std::vector& def_levels, size_t start_index = 0); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + const size_t decoded = _def_level_decoder.get_levels(ptr, append_sz); + if (UNLIKELY(decoded != append_sz)) { + def_values.resize(before_sz); + return Status::Corruption( + "Parquet definition level stream ended after {} of {} slots", decoded, + append_sz); + } + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + + Slice get_page_data() const { return _page_data; } + const Slice& v2_rep_levels() const { return _v2_rep_levels; } + const Slice& v2_def_levels() const { return _v2_def_levels; } + ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); } + +private: + enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + + // for check dict page. + Status _parse_first_page_header(); + Status _decode_dict_page(); + + void _reserve_decompress_buf(size_t size); + int32_t _get_type_length(); + void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); + + Status _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); + + level_t _rep_level_get_next() { + if constexpr (IN_COLLECTION) { + return _rep_level_decoder.get_next(); + } + return 0; + } + + void _rep_level_rewind_one() { + if constexpr (IN_COLLECTION) { + _rep_level_decoder.rewind_one(); + } + } + + ColumnChunkReaderState _state = NOT_INIT; + NativeFieldSchema* _field_schema = nullptr; + const level_t _max_rep_level; + const level_t _max_def_level; + + io::BufferedStreamReader* _stream_reader = nullptr; + tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; + io::IOContext* _io_ctx = nullptr; + + std::unique_ptr> _page_reader; + BlockCompressionCodec* _block_compress_codec = nullptr; + + ParquetPageReadContext _page_read_ctx; + ColumnChunkRange _chunk_range; + bool _has_validated_chunk_range = false; + + LevelDecoder _rep_level_decoder; + LevelDecoder _def_level_decoder; + size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. + uint32_t _remaining_num_values = 0; + + Slice _page_data; + DorisUniqueBufferPtr _decompress_buf; + size_t _decompress_buf_size = 0; + Slice _v2_rep_levels; + Slice _v2_def_levels; + bool _dict_checked = false; + bool _has_dict = false; + bool _nested_row_started = false; + Decoder* _page_decoder = nullptr; + std::unique_ptr _empty_value_decoder; + bool _empty_value_section = false; + tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; + // Map: encoding -> Decoder + // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding + std::unordered_map> _decoders; + NullMap _nullable_selection_nulls; + ColumnChunkReaderStatistics _chunk_statistics; +}; + +bool has_dict_page(const tparquet::ColumnMetaData& column); + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp new file mode 100644 index 00000000000000..9417da15683c87 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -0,0 +1,1890 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet::native { +namespace { + +ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +template +bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + DORIS_CHECK(values != nullptr); + if (values->capacity() * sizeof(T) <= max_retained_bytes) { + return false; + } + std::vector().swap(*values); + return true; +} + +size_t retained_set_bytes(const std::unordered_set& values) { + return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); +} + +Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { + if (precision <= 0 || scale < 0 || scale > precision) { + return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", + field.name, precision, scale); + } + switch (field.physical_type) { + case tparquet::Type::INT32: + if (precision <= 9) { + return Status::OK(); + } + break; + case tparquet::Type::INT64: + if (precision <= 18) { + return Status::OK(); + } + break; + case tparquet::Type::BYTE_ARRAY: + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int length = + field.parquet_schema.__isset.type_length ? field.parquet_schema.type_length : -1; + if (length > 0) { + const int64_t max_precision = (static_cast(length) * 8 - 1) * 30103 / 100000; + if (precision <= max_precision) { + return Status::OK(); + } + } + break; + } + default: + break; + } + return Status::Corruption("Parquet decimal field {} has incompatible physical type {}", + field.name, tparquet::to_string(field.physical_type)); +} + +Status validate_physical_annotation(const NativeFieldSchema& field) { + const auto& schema = field.parquet_schema; + if (field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + (!schema.__isset.type_length || schema.type_length <= 0)) { + return Status::Corruption("Parquet fixed-length field {} has invalid width {}", field.name, + schema.__isset.type_length ? schema.type_length : -1); + } + auto require = [&](bool valid, std::string_view annotation) -> Status { + if (valid) { + return Status::OK(); + } + return Status::Corruption("Parquet {} field {} is incompatible with physical type {}", + annotation, field.name, tparquet::to_string(field.physical_type)); + }; + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "string"); + } + if (logical.__isset.DECIMAL) { + return validate_decimal_physical_type(field, logical.DECIMAL.precision, + logical.DECIMAL.scale); + } + if (logical.__isset.DATE) { + return require(field.physical_type == tparquet::Type::INT32, "date"); + } + if (logical.__isset.TIME) { + const auto unit = parquet_time_unit(logical.TIME.unit); + return require( + (unit == ParquetTimeUnit::MILLIS && + field.physical_type == tparquet::Type::INT32) || + ((unit == ParquetTimeUnit::MICROS || unit == ParquetTimeUnit::NANOS) && + field.physical_type == tparquet::Type::INT64), + "time"); + } + if (logical.__isset.TIMESTAMP) { + return require( + field.physical_type == tparquet::Type::INT64 && + parquet_time_unit(logical.TIMESTAMP.unit) != ParquetTimeUnit::UNKNOWN, + "timestamp"); + } + if (logical.__isset.INTEGER) { + const int width = logical.INTEGER.bitWidth; + return require(((width == 8 || width == 16 || width == 32) && + field.physical_type == tparquet::Type::INT32) || + (width == 64 && field.physical_type == tparquet::Type::INT64), + "integer"); + } + if (logical.__isset.UUID) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 16, + "UUID"); + } + if (logical.__isset.FLOAT16) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 2, + "FLOAT16"); + } + if (logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + // Geospatial WKB has no Doris logical mapping here; keep its BYTE_ARRAY payload raw so + // reader validation preserves the physical fallback chosen by native schema inference. + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "geospatial"); + } + if (logical.__isset.UNKNOWN) { + return Status::OK(); + } + return Status::Corruption("Unsupported Parquet logical annotation on field {}", field.name); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "converted string"); + case tparquet::ConvertedType::DECIMAL: + return validate_decimal_physical_type(field, + schema.__isset.precision ? schema.precision : -1, + schema.__isset.scale ? schema.scale : -1); + case tparquet::ConvertedType::DATE: + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + return require(field.physical_type == tparquet::Type::INT32, "converted INT32"); + case tparquet::ConvertedType::TIME_MICROS: + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_64: + return require(field.physical_type == tparquet::Type::INT64, "converted INT64"); + case tparquet::ConvertedType::INTERVAL: + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 12, + "interval"); + default: + return Status::Corruption("Unsupported Parquet converted annotation {} on field {}", + tparquet::to_string(schema.converted_type), field.name); + } +} + +IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn::Filter* output_null_map, + IColumn::Filter* compatibility_scratch) { + const auto& schema = field.parquet_schema; + const bool is_utc_timestamp = + field.physical_type == tparquet::Type::INT96 || + (field.physical_type == tparquet::Type::INT64 && + ((schema.__isset.logicalType && schema.logicalType.__isset.TIMESTAMP && + schema.logicalType.TIMESTAMP.isAdjustedToUTC) || + (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS || + schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MICROS)))); + if (!strict_mode && output_null_map != nullptr && is_utc_timestamp && + remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2) { + // Legacy UTC timestamp conversion kept out-of-range values as DATETIME defaults. Local + // timestamps intentionally keep non-strict NULL-on-overflow behavior because timezone + // conversion is not part of their representation. + compatibility_scratch->resize_fill(output_null_map->size(), 0); + return compatibility_scratch; + } + return output_null_map; +} + +void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTypePtr& target_type, + bool strict_mode, IColumn& data_column, + IColumn::Filter* output_null_map, size_t start_row) { + const auto& schema = field.parquet_schema; + const bool is_local_timestamp = + field.physical_type == tparquet::Type::INT64 && schema.__isset.logicalType && + schema.logicalType.__isset.TIMESTAMP && !schema.logicalType.TIMESTAMP.isAdjustedToUTC; + if (strict_mode || output_null_map == nullptr || !is_local_timestamp || + remove_nullable(target_type)->get_primitive_type() != TYPE_DATETIMEV2) { + return; + } + auto& values = assert_cast(data_column).get_data(); + DORIS_CHECK_EQ(values.size(), output_null_map->size()); + for (size_t row = start_row; row < values.size(); ++row) { + if ((*output_null_map)[row] == 0 && !values[row].is_valid_date()) { + // Local timestamps before Doris' representable calendar can materialize as a zero + // date without a SerDe error. Preserve non-strict scan semantics by nulling only that + // sentinel; physical NULLs and valid local timestamps keep their original map bits. + (*output_null_map)[row] = 1; + } + } +} + +Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + DORIS_CHECK(context != nullptr); + // Annotation validation belongs before decoder construction: accepting an impossible pair + // lets the logical SerDe reinterpret a differently sized physical value stream. + RETURN_IF_ERROR(validate_physical_annotation(field)); + switch (field.physical_type) { + case tparquet::Type::BOOLEAN: + context->physical_type = ParquetPhysicalType::BOOLEAN; + break; + case tparquet::Type::INT32: + context->physical_type = ParquetPhysicalType::INT32; + break; + case tparquet::Type::INT64: + context->physical_type = ParquetPhysicalType::INT64; + break; + case tparquet::Type::INT96: + context->physical_type = ParquetPhysicalType::INT96; + break; + case tparquet::Type::FLOAT: + context->physical_type = ParquetPhysicalType::FLOAT; + break; + case tparquet::Type::DOUBLE: + context->physical_type = ParquetPhysicalType::DOUBLE; + break; + case tparquet::Type::BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::BYTE_ARRAY; + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY; + break; + default: + return Status::NotSupported("Unsupported Parquet physical type {}", + tparquet::to_string(field.physical_type)); + } + + const auto& schema = field.parquet_schema; + context->type_length = schema.__isset.type_length ? schema.type_length : -1; + context->decimal_precision = schema.__isset.precision ? schema.precision : -1; + context->decimal_scale = schema.__isset.scale ? schema.scale : -1; + context->timezone = ctz; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + context->logical_type = ParquetLogicalType::STRING; + } else if (logical.__isset.DECIMAL) { + context->logical_type = ParquetLogicalType::DECIMAL; + context->decimal_precision = logical.DECIMAL.precision; + context->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.DATE) { + context->logical_type = ParquetLogicalType::DATE; + } else if (logical.__isset.TIME) { + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = parquet_time_unit(logical.TIME.unit); + } else if (logical.__isset.TIMESTAMP) { + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = parquet_time_unit(logical.TIMESTAMP.unit); + context->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + } else if (logical.__isset.INTEGER) { + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_bit_width = logical.INTEGER.bitWidth; + context->logical_integer_is_signed = logical.INTEGER.isSigned; + } else if (logical.__isset.UUID) { + context->logical_type = ParquetLogicalType::UUID; + context->logical_uuid = true; + } else if (logical.__isset.FLOAT16) { + context->logical_type = ParquetLogicalType::FLOAT16; + context->logical_float16 = true; + } + if (context->logical_uuid && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 16)) { + return Status::Corruption("Parquet UUID field {} must be FIXED_LEN_BYTE_ARRAY(16)", + field.name); + } + if (context->logical_float16 && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 2)) { + return Status::Corruption("Parquet FLOAT16 field {} must be FIXED_LEN_BYTE_ARRAY(2)", + field.name); + } + return Status::OK(); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + context->logical_type = ParquetLogicalType::STRING; + break; + case tparquet::ConvertedType::DECIMAL: + context->logical_type = ParquetLogicalType::DECIMAL; + break; + case tparquet::ConvertedType::DATE: + context->logical_type = ParquetLogicalType::DATE; + break; + case tparquet::ConvertedType::TIME_MILLIS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIME_MICROS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MILLIS; + // Legacy converted timestamps are defined as UTC-adjusted, unlike an unannotated INT64. + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::TIMESTAMP_MICROS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MICROS; + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::INT_64: + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_is_signed = + schema.converted_type >= tparquet::ConvertedType::INT_8; + context->logical_integer_bit_width = + schema.converted_type == tparquet::ConvertedType::UINT_8 || + schema.converted_type == tparquet::ConvertedType::INT_8 + ? 8 + : schema.converted_type == tparquet::ConvertedType::UINT_16 || + schema.converted_type == tparquet::ConvertedType::INT_16 + ? 16 + : schema.converted_type == tparquet::ConvertedType::UINT_32 || + schema.converted_type == tparquet::ConvertedType::INT_32 + ? 32 + : 64; + break; + default: + break; + } + return Status::OK(); +} + +} // namespace + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + return init_decode_context(field, ctz, context); +} + +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode) { + IColumn::Filter output_null_map; + output_null_map.resize_fill(1, 0); + IColumn::Filter compatibility_scratch; + return conversion_failure_map(field, target_type, strict_mode, &output_null_map, + &compatibility_scratch) == &compatibility_scratch; +} + +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row) { + mark_local_timestamp_defaults(field, target_type, strict_mode, data_column, output_null_map, + start_row); +} +#endif + +static void fill_struct_null_map(NativeFieldSchema* field, NullMap& null_map, + const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = def_levels.size(); + DCHECK_EQ(num_levels, rep_levels.size()); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_levels); + size_t pos = origin_size; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (def_levels[i] >= field->definition_level) { + null_map[pos++] = 0; + } else { + null_map[pos++] = 1; + } + } + null_map.resize(pos); +} + +static Status fill_array_offset(NativeFieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = rep_levels.size(); + if (UNLIKELY(num_levels != def_levels.size())) { + return Status::Corruption("Parquet repetition and definition level counts differ"); + } + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_levels); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(origin_size + num_levels); + } + size_t offset_pos = origin_size - 1; + bool parent_opened = false; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (rep_levels[i] == field->repetition_level) { + // A continuation can extend only a parent opened by this aligned logical batch. + if (UNLIKELY(!parent_opened)) { + return Status::Corruption( + "Parquet collection starts with an orphan repetition continuation"); + } + offsets_data[offset_pos]++; + continue; + } + parent_opened = true; + offset_pos++; + offsets_data[offset_pos] = offsets_data[offset_pos - 1]; + if (def_levels[i] >= field->definition_level) { + offsets_data[offset_pos]++; + } + if (null_map_ptr != nullptr) { + if (def_levels[i] >= field->definition_level - 1) { + (*null_map_ptr)[offset_pos] = 0; + } else { + (*null_map_ptr)[offset_pos] = 1; + } + } + } + offsets_data.resize(offset_pos + 1); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(offset_pos + 1); + } + return Status::OK(); +} + +Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection, + const std::set& column_ids, + const std::set& filter_column_ids, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode) { + size_t total_rows = row_group.num_rows; + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + std::unique_ptr element_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + element_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, enable_strict_mode)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + element_reader->set_column_in_nested(); + RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); + array_reader->_filter_column_ids = filter_column_ids; + reader.reset(array_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + std::unique_ptr key_reader; + std::unique_ptr value_reader; + + if (column_ids.empty() || + column_ids.find(field->children[0].get_column_id()) != column_ids.end()) { + // Create key reader + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + key_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[0]); + key_reader = std::move(skip_reader); + } + + if (column_ids.empty() || + column_ids.find(field->children[1].get_column_id()) != column_ids.end()) { + // Create value reader + RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, + value_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[1]); + value_reader = std::move(skip_reader); + } + + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + key_reader->set_column_in_nested(); + value_reader->set_column_in_nested(); + RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); + map_reader->_filter_column_ids = filter_column_ids; + reader.reset(map_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + std::unordered_map> child_readers; + child_readers.reserve(field->children.size()); + int non_skip_reader_idx = -1; + for (int i = 0; i < field->children.size(); ++i) { + auto& child = field->children[i]; + std::unique_ptr child_reader; + if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { + RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, + in_collection, column_ids, filter_column_ids, + page_cache_file_key, compat, enable_strict_mode)); + child_readers[child.name] = std::move(child_reader); + // Record the first non-SkippingReader + if (non_skip_reader_idx == -1) { + non_skip_reader_idx = i; + } + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &child); + skip_reader->_filter_column_ids = filter_column_ids; + child_readers[child.name] = std::move(skip_reader); + } + child_readers[child.name]->set_column_in_nested(); + } + // If all children are SkipReadingReader, force the first child to call create + if (non_skip_reader_idx == -1) { + std::unique_ptr child_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, in_collection, + column_ids, filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + child_reader->set_column_in_nested(); + child_readers[field->children[0].name] = std::move(child_reader); + } + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); + struct_reader->_filter_column_ids = filter_column_ids; + reader.reset(struct_reader.release()); + } else { + auto physical_index = field->physical_column_index; + if (physical_index < 0 || static_cast(physical_index) >= row_group.columns.size()) { + // Keep the leaf-to-chunk invariant checked at this consumer too because unit callers + // can construct a reader without going through NativeParquetMetadata::init_schema(). + return Status::Corruption("Parquet physical column index {} is out of range {}", + physical_index, row_group.columns.size()); + } + const auto offset_it = col_offsets.find(physical_index); + const tparquet::OffsetIndex* offset_index = + offset_it != col_offsets.end() ? &offset_it->second : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet physical column {} has no chunk metadata", + physical_index); + } + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } + } + return Status::OK(); +} + +void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const { + result_ranges->add(page_row_range); + RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges); +} + +template +Status ScalarColumnReader::init( + io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + bool enable_strict_mode) { + _field_schema = field; + auto& chunk_meta = _chunk_meta.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(chunk_meta, file->size(), compat.parquet_816_padding, + &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_len = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); + if ((typeid_cast(file.get()) && + typeid_cast( + ((doris::io::TracingFileReader*)(file.get()))->inner_reader().get())) || + typeid_cast(file.get())) { + // turn off prefetch data when using MergeRangeFileReader + prefetch_buffer_size = 0; + } + _stream_reader = std::make_unique(file, chunk_start, chunk_len, + prefetch_buffer_size); + ParquetPageReadContext ctx( + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, + page_cache_file_key, compat.data_page_v2_always_compressed); + + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx, + &chunk_range); + _materialization_state.enable_strict_mode = enable_strict_mode; + RETURN_IF_ERROR(_chunk_reader->init()); + RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); + return Status::OK(); +} + +template +void ScalarColumnReader::release_batch_scratch( + size_t max_retained_bytes) { + const size_t retained_bytes = retained_batch_scratch_bytes(); + const size_t active_bytes = active_batch_scratch_bytes(); + if (retained_bytes <= max_retained_bytes || active_bytes > max_retained_bytes) { + _oversized_scratch_idle_batches = 0; + return; + } + // An adaptive probe or one repeated outlier must not pin memory forever, but immediately + // dropping a large steady-state buffer makes every following batch allocate it again. Require + // three ordinary batches before treating an oversized capacity as idle. + constexpr uint8_t OVERSIZED_SCRATCH_IDLE_BATCHES = 3; + if (++_oversized_scratch_idle_batches < OVERSIZED_SCRATCH_IDLE_BATCHES) { + return; + } + _oversized_scratch_idle_batches = 0; + if (_chunk_reader != nullptr) { + // Persistent decoders also own batch-sized value/slice buffers, not only the reader. + _chunk_reader->release_decoder_scratch(max_retained_bytes); + } + bool release_selection = false; + release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_null_run_lengths, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, + max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.selection.ranges, + max_retained_bytes); + if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { + std::unordered_set().swap(_ancestor_null_indices); + release_selection = true; + } + if (release_selection) { + _select_vector = ColumnSelectVector(); + } +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->retained_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.capacity() * sizeof(level_t) + + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + + retained_set_bytes(_ancestor_null_indices); +} + +template +size_t ScalarColumnReader::active_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->active_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.size() * sizeof(level_t) + + _def_levels.size() * sizeof(level_t) + _null_run_lengths.size() * sizeof(uint16_t) + + _nested_filter_map_data.size() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.size() * sizeof(uint32_t) + + _materialization_state.selection.ranges.size() * sizeof(ParquetSelectionRange) + + _ancestor_null_indices.size() * sizeof(size_t); +} + +#ifdef BE_TEST +template +void ScalarColumnReader::reserve_batch_scratch_for_test( + size_t elements) { + _rep_levels.reserve(elements); + _def_levels.reserve(elements); + _null_run_lengths.reserve(elements); + _nested_filter_map_data.reserve(elements); + _materialization_state.dictionary_indices.reserve(elements); + _materialization_state.selection.ranges.reserve(elements); + _ancestor_null_indices.reserve(elements); +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() + const { + return retained_batch_scratch_bytes(); +} +#endif + +template +Status ScalarColumnReader::_skip_values(size_t num_values) { + if (num_values == 0) { + return Status::OK(); + } + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t skipped = 0; + size_t null_size = 0; + size_t nonnull_size = 0; + while (skipped < num_values) { + level_t def_level = -1; + size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); + if (loop_skip == 0) { + return Status::Corruption("Parquet definition level stream ended while skipping"); + } + if (def_level < _field_schema->definition_level) { + null_size += loop_skip; + } else { + nonnull_size += loop_skip; + } + skipped += loop_skip; + } + if (null_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); + } + if (nonnull_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); + } + } else { + RETURN_IF_ERROR(_chunk_reader->skip_values(num_values)); + } + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + const DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { + if (num_values == 0) { + return Status::OK(); + } + MutableColumnPtr data_column; + _null_run_lengths.clear(); + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level; + size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while materializing"); + } + + bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + prev_is_null = is_null; + has_read += loop_read; + } + } + } else { + if (_chunk_reader->max_def_level() > 0) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (_null_run_lengths.empty()) { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + } + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, map_data_column, + &filter_map, _filter_map_index)); + _filter_map_index += num_values; + } + DORIS_CHECK(_serde != nullptr); + // Keep selected-row cardinality stable: non-strict conversion failures append a nested + // default and mark this matching nullable output row instead of shortening the column. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = + conversion_failure_map(*_field_schema, type, _materialization_state.enable_strict_mode, + map_data_column, &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + return status; +} + +/** + * Load the nested column data of complex type. + * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that + * whether the reader should read the remaining value of the last row in previous page. + */ +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); + + // Handle nullable columns + MutableColumnPtr data_column; + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + _nested_filter_map_data.clear(); + + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + } else { + RETURN_IF_ERROR(_nested_filter_map.init( + nullptr, _rep_levels.size() - before_rep_level_sz, false)); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), + _null_run_lengths, _ancestor_null_indices)); + + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init( + _null_run_lengths, + _rep_levels.size() - before_rep_level_sz - _ancestor_null_indices.size(), + map_data_column, &_nested_filter_map, 0, &_ancestor_null_indices)); + } + + DORIS_CHECK(_serde != nullptr); + // Nested materialization must preserve the same value/null-map row alignment invariant. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = conversion_failure_map( + *_field_schema, type, _materialization_state.enable_strict_mode, map_data_column, + &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values( + data_column, *_serde, _decode_context, _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + RETURN_IF_ERROR(status); + if (!_ancestor_null_indices.empty()) { + RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (_nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; + } + } + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges.get_range_to(_current_range_idx)); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && load_rows == 0 && !cross_page)) { + // A bounded V2/indexed page must advance at least one logical row; zero progress would + // leave both range cursors unchanged and spin forever on corrupt repetition levels. + return Status::Corruption("Parquet nested reader made no row progress"); + } + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_fixed_width_filter_values( + size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter) { + DORIS_CHECK(row_filter != nullptr); + _null_run_lengths.clear(); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level = -1; + const size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while filtering fixed-width values"); + } + const bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + prev_is_null = is_null; + has_read += loop_read; + } + } else { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + } + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, nullptr, &filter_map, + _filter_map_index)); + _filter_map_index += num_values; + bool used_filter = false; + RETURN_IF_ERROR(_chunk_reader->filter_fixed_width_values( + conjuncts, column_id, _select_vector, &_fixed_width_predicate_nulls, + &_fixed_width_predicate_matches, projected_column, row_filter, &used_filter)); + // Chunk encodings are prevalidated before any definition level is consumed, so a false result + // here would make a materializing fallback observe an advanced level cursor. + DORIS_CHECK(used_filter); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_fixed_width_filter( + const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + if (_in_nested || conjuncts.empty()) { + return Status::OK(); + } + if (!std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + })) { + return Status::OK(); + } + // Validate every advertised value encoding before touching either cursor. A late fallback + // cannot rewind definition levels or a previously decoded fixed-width page. + const bool supported_encodings = std::ranges::all_of( + _chunk_meta.meta_data.encodings, [&](const tparquet::Encoding::type encoding) { + return ColumnChunkReader:: + supports_raw_fixed_filter_encoding(encoding, + _chunk_meta.meta_data.type) || + encoding == tparquet::Encoding::RLE || + encoding == tparquet::Encoding::BIT_PACKED; + }); + if (!supported_encodings) { + return Status::OK(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + _current_row_index = right_row; + } else { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + if (!_chunk_reader->can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); ++idx) { + const auto range = read_ranges.get_range(idx); + const size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + const size_t values = + std::min(static_cast(range.to() - range.from()), batch_size - has_read); + IColumn::Filter fragment_filter; + RETURN_IF_ERROR(_read_fixed_width_filter_values( + values, conjuncts, column_id, filter_map, projected_column, &fragment_filter)); + row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); + has_read += values; + *read_rows += values; + _current_row_index += values; + if (has_read == batch_size) { + break; + } + } + } + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + *used_filter = true; + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, + size_t batch_size, + size_t* read_rows, + bool* eof) { + DORIS_CHECK(_in_nested); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + _rep_levels.clear(); + _def_levels.clear(); + *read_rows = 0; + + auto consume_level_segment = [&](size_t level_start, size_t filter_map_index) -> Status { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + // Advance the encoded value stream without constructing a temporary Doris column. The + // definition levels identify the physical values that actually exist; dictionary skips + // still validate every index. + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(_def_levels, level_start)); + if (!filter_map.has_filter()) { + return Status::OK(); + } + + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, level_start, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + size_t write_index = level_start; + for (size_t read_index = level_start; read_index < _rep_levels.size(); ++read_index) { + if (_nested_filter_map_data[read_index - level_start] != 0) { + _rep_levels[write_index] = _rep_levels[read_index]; + _def_levels[write_index] = _def_levels[read_index]; + ++write_index; + } + } + _rep_levels.resize(write_index); + _def_levels.resize(write_index); + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + const size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + const size_t right_row = + std::min(left_row + batch_size - *read_rows, + static_cast(_row_ranges.get_range_to(_current_range_idx))); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + + size_t loaded_rows = 0; + bool cross_page = false; + size_t level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &loaded_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && loaded_rows == 0 && !cross_page)) { + // Keep the levels-only path under the same forward-progress invariant as value reads. + return Status::Corruption("Parquet nested level reader made no row progress"); + } + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); + _filter_map_index += loaded_rows; + while (cross_page) { + level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index - 1)); + } + + *read_rows += loaded_rows; + _current_row_index += loaded_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Result +ScalarColumnReader::materialize_dictionary_values( + const ColumnInt32* dict_column, const DataTypePtr& target_type) { + DORIS_CHECK(dict_column != nullptr); + DORIS_CHECK(target_type != nullptr); + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + DORIS_CHECK(dictionary_decoder != nullptr); + // Materialize the dictionary in the file-local logical type once. Both predicate evaluation + // and matched-ID gathering must observe the same decimal scale, timestamp unit, and binary + // interpretation as ordinary data-page decoding. + const DataTypePtr dictionary_type = remove_nullable(target_type); + const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + if (_materialization_state.dictionary_generation != + dictionary_decoder->dictionary_generation()) { + _materialization_state.typed_dictionary = dictionary_type->create_column(); + ParquetDecodeContext dictionary_context = _decode_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + dictionary_context.dictionary_index_only = false; + auto status = dictionary_serde->read_parquet_dictionary( + *_materialization_state.typed_dictionary, *dictionary_decoder, dictionary_context); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), + dictionary_decoder->dictionary_size()); + _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); + } + + auto result = _materialization_state.typed_dictionary->clone_empty(); + const auto& source_indices = dict_column->get_data(); + auto& indices = _materialization_state.dictionary_indices; + indices.resize(source_indices.size()); + for (size_t row = 0; row < source_indices.size(); ++row) { + if (UNLIKELY(source_indices[row] < 0 || + static_cast(source_indices[row]) >= + _materialization_state.typed_dictionary->size())) { + return ResultError(Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + source_indices[row], row, _materialization_state.typed_dictionary->size())); + } + indices[row] = static_cast(source_indices[row]); + } + result->insert_indices_from(*_materialization_state.typed_dictionary, indices.data(), + indices.data() + indices.size()); + return result; +} + +template +Result ScalarColumnReader::dictionary_values( + const DataTypePtr& target_type) { + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { + return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); + } + auto ids = ColumnInt32::create(); + auto& data = ids->get_data(); + data.resize(dictionary_decoder->dictionary_size()); + for (size_t dictionary_id = 0; dictionary_id < data.size(); ++dictionary_id) { + data[dictionary_id] = cast_set(dictionary_id); + } + // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level + // filtering decodes only ids and flattens surviving values from this same dictionary. + return materialize_dictionary_values(ids.get(), target_type); +} + +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + const DataTypePtr materialization_type = remove_nullable(type); + const DataTypePtr file_type = remove_nullable(_field_schema->data_type); + if (!is_dict_filter && !file_type->equals(*materialization_type)) { + // File-to-table casts belong to ColumnMapper. Reaching this reader with a table type would + // also evaluate file-local predicates against values produced under the wrong type. + return Status::InternalError( + "Native Parquet reader type mismatch for file column '{}': file={}, requested={}", + _field_schema->name, file_type->get_name(), materialization_type->get_name()); + } + + const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; + if (_serde_type != serde_type.get() || _dictionary_index_only != is_dict_filter) { + _serde_type = serde_type.get(); + _serde = serde_type->get_serde(); + _dictionary_index_only = is_dict_filter; + _materialization_state.reset_dictionary(); + } + _decode_context.dictionary_index_only = is_dict_filter; + + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if (_in_nested) { + return _read_nested_column(doris_column, materialization_type, filter_map, batch_size, + read_rows, eof, is_dict_filter); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + + do { + // generate the row ranges that should be read + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + // skip the whole page + _current_row_index = right_row; + } else { + bool skip_whole_batch = false; + // Determining whether to skip page or batch will increase the calculation time. + // When the filtering effect is greater than 60%, it is possible to skip the page or batch. + if (filter_map.has_filter() && filter_map.filter_ratio() > 0.6) { + // lazy read + size_t remaining_num_values = read_ranges.count(); + if (batch_size >= remaining_num_values && + filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { + // We can skip the whole page if the remaining values are filtered by predicate columns + _filter_map_index += remaining_num_values; + _current_row_index = right_row; + *read_rows = remaining_num_values; + break; + } + skip_whole_batch = batch_size <= remaining_num_values && + filter_map.can_filter_all(batch_size, _filter_map_index); + if (skip_whole_batch) { + _filter_map_index += batch_size; + } + } + // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); idx++) { + auto range = read_ranges.get_range(idx); + // generate the skipped values + size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + // generate the read values + size_t read_values = + std::min((size_t)(range.to() - range.from()), batch_size - has_read); + if (skip_whole_batch) { + RETURN_IF_ERROR(_skip_values(read_values)); + } else { + RETURN_IF_ERROR(_read_values(read_values, doris_column, materialization_type, + filter_map, is_dict_filter)); + } + has_read += read_values; + *read_rows += read_values; + _current_row_index += read_values; + if (has_read == batch_size) { + break; + } + } + } + } while (false); + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + + return Status::OK(); +} + +Status ArrayColumnReader::init(std::unique_ptr element_reader, + NativeFieldSchema* field) { + _field_schema = field; + _element_reader = std::move(element_reader); + return Status::OK(); +} + +Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_ARRAY) { + return Status::Corruption( + "Wrong data type for column '{}', expected Array type, actual type: {}.", + _field_schema->name, type->get_name()); + } + + ColumnPtr& element_column = assert_cast(*data_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(type).get()))->get_nested_type(); + // read nested column + RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, + root_node->element(), filter_map, batch_size, + read_rows, eof, is_dict_filter)); + if (*read_rows == 0) { + return Status::OK(); + } + + ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); + // fill offset and null map + RETURN_IF_ERROR(fill_array_offset(_field_schema, offsets_data, null_map_ptr, + _element_reader->get_rep_level(), + _element_reader->get_def_level())); + if (UNLIKELY(element_column->size() != offsets_data.back())) { + return Status::Corruption("Parquet array element count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field) { + _field_schema = field; + _key_reader = std::move(key_reader); + _value_reader = std::move(value_reader); + return Status::OK(); +} + +Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_MAP) { + return Status::Corruption( + "Wrong data type for column '{}', expected Map type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& map = assert_cast(*data_column); + const DataTypePtr& key_type = + assert_cast(remove_nullable(type).get())->get_key_type(); + const DataTypePtr& value_type = + assert_cast(remove_nullable(type).get())->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + + size_t key_rows = 0; + size_t value_rows = 0; + bool key_eof = false; + bool value_eof = false; + int64_t orig_col_column_size = key_column->size(); + + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->key(), + filter_map, batch_size, &key_rows, &key_eof, + is_dict_filter)); + + while (value_rows < key_rows && !value_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_value_reader->read_column_data( + value_column, value_type, root_node->value(), filter_map, key_rows - value_rows, + &loop_rows, &value_eof, is_dict_filter, key_column->size() - orig_col_column_size)); + value_rows += loop_rows; + } + if (UNLIKELY(key_rows != value_rows)) { + // MAP children share one logical-row boundary; EOF in one sibling is file corruption. + return Status::Corruption("Parquet map value reader returned {} rows for {} key rows", + value_rows, key_rows); + } + *read_rows = key_rows; + *eof = key_eof; + + if (*read_rows == 0) { + return Status::OK(); + } + + const auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + // MAP siblings may have child-local collections and nullness. At this boundary, + // only the outer entry distribution and whether that entry exists must agree. + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + if (UNLIKELY(_key_reader->get_rep_level().size() != _key_reader->get_def_level().size() || + _value_reader->get_rep_level().size() != _value_reader->get_def_level().size())) { + return Status::Corruption("Parquet map sibling level counts differ"); + } + if (UNLIKELY(parent_shape(*_key_reader) != parent_shape(*_value_reader))) { + return Status::Corruption("Parquet map key/value outer entry shapes differ"); + } + + if (UNLIKELY(key_column->size() != value_column->size())) { + return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", + key_column->size(), value_column->size()); + } + // Non-strict conversion can persist nullable Doris MAP keys, so the native reader must + // preserve writer output instead of reclassifying an otherwise valid file as corrupt. + const auto& key_rep_levels = _key_reader->get_rep_level(); + // fill offset and null map + // The key leaf is the canonical outer MAP shape. A nested value has additional repetition + // levels for its own collections, so comparing the two leaves would reject valid MAP values. + RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, + key_rep_levels, _key_reader->get_def_level())); + if (UNLIKELY(key_column->size() != map.get_offsets().back())) { + return Status::Corruption("Parquet map entry count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + DORIS_CHECK(dynamic_cast(_key_reader.get()) == nullptr); + return _key_reader->read_column_levels(filter_map, batch_size, read_rows, eof); +} + +Status StructColumnReader::init( + std::unordered_map>&& child_readers, + NativeFieldSchema* field) { + _field_schema = field; + _child_readers = std::move(child_readers); + return Status::OK(); +} + +Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + _read_column_names.clear(); + for (const auto& child : _field_schema->children) { + auto reader = _child_readers.find(child.name); + DORIS_CHECK(reader != _child_readers.end()); + if (dynamic_cast(reader->second.get()) != nullptr) { + continue; + } + _read_column_names.emplace_back(child.name); + return reader->second->read_column_levels(filter_map, batch_size, read_rows, eof); + } + return Status::InternalError("Struct {} has no physical reader for levels", + _field_schema->name); +} +Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_STRUCT) { + return Status::Corruption( + "Wrong data type for column '{}', expected Struct type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = assert_cast(remove_nullable(type).get()); + + int64_t not_missing_column_id = -1; + size_t not_missing_orig_column_size = 0; + std::vector missing_column_idxs {}; + std::vector skip_reading_column_idxs {}; + std::vector> reference_parent_shape; + + auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's + // boundary and the optional parent's presence determine how siblings are paired. + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + + _read_column_names.clear(); + + for (size_t i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& doris_field = doris_struct.get_column_ptr(i); + auto& doris_type = doris_struct_type->get_element(i); + auto& doris_name = doris_struct_type->get_element_name(i); + if (!root_node->has_child(doris_name)) { + missing_column_idxs.push_back(i); + VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i + << "], doris_name: " << doris_name << " (column not exists in root node)"; + continue; + } + auto file_name = root_node->file_child_name(doris_name); + + // Check if this is a SkipReadingReader - we should skip it when choosing reference column + // because SkipReadingReader doesn't know the actual data size in nested context + bool is_skip_reader = + dynamic_cast(_child_readers[file_name].get()) != nullptr; + + if (is_skip_reader) { + // Store SkipReadingReader columns to fill them later based on reference column size + skip_reading_column_idxs.push_back(i); + continue; + } + + // Only add non-SkipReadingReader columns to _read_column_names + // This ensures get_rep_level() and get_def_level() return valid levels + _read_column_names.emplace_back(file_name); + + size_t field_rows = 0; + bool field_eof = false; + if (not_missing_column_id == -1) { + not_missing_column_id = i; + not_missing_orig_column_size = doris_field->size(); + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, batch_size, + &field_rows, &field_eof, is_dict_filter)); + *read_rows = field_rows; + *eof = field_eof; + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + reference_parent_shape = parent_shape(*_child_readers[file_name]); + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column + * (since `_read_nested_column` is not called repeatedly). + * + * It is worth mentioning that, theoretically, any sub-column can be chosen to fill the null_map, + * and selecting the shortest one will offer better performance + */ + } else { + while (field_rows < *read_rows && !field_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + *read_rows - field_rows, &loop_rows, &field_eof, is_dict_filter)); + field_rows += loop_rows; + } + if (UNLIKELY(*read_rows != field_rows)) { + // STRUCT siblings must advance the same logical rows before any result is exposed. + return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", + file_name, field_rows, *read_rows); + } + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { + return Status::Corruption( + "Parquet struct child '{}' has a different repeated-parent shape", + file_name); + } + // DCHECK_EQ(*eof, field_eof); + } + } + + int64_t missing_column_sz = -1; + + if (not_missing_column_id == -1) { + // All queried columns are missing in the file (e.g., all added after schema change) + // We need to pick a column from _field_schema children that exists in the file for RL/DL reference + std::string reference_file_column_name; + std::unique_ptr* reference_reader = nullptr; + + for (const auto& child : _field_schema->children) { + auto it = _child_readers.find(child.name); + if (it != _child_readers.end()) { + // Skip SkipReadingReader as they don't have valid RL/DL + bool is_skip_reader = dynamic_cast(it->second.get()) != nullptr; + if (!is_skip_reader) { + reference_file_column_name = child.name; + reference_reader = &(it->second); + break; + } + } + } + + if (reference_reader != nullptr) { + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR( + (*reference_reader) + ->read_column_levels(filter_map, batch_size, &field_rows, &field_eof)); + + *read_rows = field_rows; + *eof = field_eof; + _read_column_names.emplace_back(reference_file_column_name); + missing_column_sz = 0; + const auto& rep_levels = (*reference_reader)->get_rep_level(); + const auto& def_levels = (*reference_reader)->get_def_level(); + DORIS_CHECK_EQ(rep_levels.size(), def_levels.size()); + for (size_t level_index = 0; level_index < def_levels.size(); ++level_index) { + if (def_levels[level_index] >= _field_schema->repeated_parent_def_level && + rep_levels[level_index] <= _field_schema->repetition_level) { + ++missing_column_sz; + } + } + } else { + return Status::Corruption( + "Cannot read struct '{}': all queried columns are missing and no reference " + "column found in file", + _field_schema->name); + } + } + + // This missing_column_sz is not *read_rows. Because read_rows returns the number of rows. + // For example: suppose we have a column array>, + // where b is a newly added column, that is, a missing column. + // There are two rows of data in this column, + // [{1,null},{2,null},{3,null}] + // [{4,null},{5,null}] + // When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2. + // You should insert 5 records into subcolumn b instead of 2. + if (missing_column_sz == -1) { + missing_column_sz = doris_struct.get_column(not_missing_column_id).size() - + not_missing_orig_column_size; + } + + // Fill SkipReadingReader columns with the correct amount of data based on reference column + // Let SkipReadingReader handle the data filling through its read_column_data method + for (auto idx : skip_reading_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = const_cast(doris_struct_type->get_element(idx)); + auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); + auto file_name = root_node->file_child_name(doris_name); + + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); + } + + // Fill truly missing columns (not in root_node) with null or default value + for (auto idx : missing_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = doris_struct_type->get_element(idx); + DCHECK(doris_type->is_nullable()); + doris_field = IColumn::mutate(std::move(doris_field)); + auto mutable_column = doris_field->assert_mutable(); + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } + + if (null_map_ptr != nullptr) { + fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(), + this->get_def_level()); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h new file mode 100644 index 00000000000000..b9a25a8132ed91 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -0,0 +1,656 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/common.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "storage/segment/row_ranges.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { +using ::doris::ColumnString; +using segment_v2::RowRange; +using segment_v2::RowRanges; + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context); +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode); +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row); +#endif + +class ColumnReader { +public: + struct ColumnStatistics { + ColumnStatistics() + : page_index_read_calls(0), + decompress_time(0), + decompress_cnt(0), + decode_header_time(0), + decode_value_time(0), + materialization_time(0), + hybrid_selection_batches(0), + hybrid_selection_ranges(0), + hybrid_selection_null_fallback_batches(0), + decode_dict_time(0), + decode_level_time(0), + decode_null_map_time(0), + skip_page_header_num(0), + parse_page_header_num(0), + read_page_header_time(0), + page_read_counter(0), + page_cache_write_counter(0), + page_cache_compressed_write_counter(0), + page_cache_decompressed_write_counter(0), + page_cache_hit_counter(0), + page_cache_missing_counter(0), + page_cache_compressed_hit_counter(0), + page_cache_decompressed_hit_counter(0) {} + + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time) + : page_index_read_calls(0), + decompress_time(cs.decompress_time), + decompress_cnt(cs.decompress_cnt), + decode_header_time(cs.decode_header_time), + decode_value_time(cs.decode_value_time), + materialization_time(cs.materialization_time), + hybrid_selection_batches(cs.hybrid_selection_batches), + hybrid_selection_ranges(cs.hybrid_selection_ranges), + hybrid_selection_null_fallback_batches(cs.hybrid_selection_null_fallback_batches), + decode_dict_time(cs.decode_dict_time), + decode_level_time(cs.decode_level_time), + decode_null_map_time(null_map_time), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num), + read_page_header_time(cs.read_page_header_time), + page_read_counter(cs.page_read_counter), + page_cache_write_counter(cs.page_cache_write_counter), + page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter), + page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter), + page_cache_hit_counter(cs.page_cache_hit_counter), + page_cache_missing_counter(cs.page_cache_missing_counter), + page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter), + leaf_page_read_counters {cs.data_page_read_counter} {} + + int64_t page_index_read_calls; + int64_t decompress_time; + int64_t decompress_cnt; + int64_t decode_header_time; + int64_t decode_value_time; + int64_t materialization_time; + int64_t hybrid_selection_batches; + int64_t hybrid_selection_ranges; + int64_t hybrid_selection_null_fallback_batches; + int64_t decode_dict_time; + int64_t decode_level_time; + int64_t decode_null_map_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; + int64_t read_page_header_time; + int64_t page_read_counter; + int64_t page_cache_write_counter; + int64_t page_cache_compressed_write_counter; + int64_t page_cache_decompressed_write_counter; + int64_t page_cache_hit_counter; + int64_t page_cache_missing_counter; + int64_t page_cache_compressed_hit_counter; + int64_t page_cache_decompressed_hit_counter; + // Preserve per-leaf identity when complex readers aggregate their other counters. + std::vector leaf_page_read_counters; + + void merge(ColumnStatistics& col_statistics) { + page_index_read_calls += col_statistics.page_index_read_calls; + decompress_time += col_statistics.decompress_time; + decompress_cnt += col_statistics.decompress_cnt; + decode_header_time += col_statistics.decode_header_time; + decode_value_time += col_statistics.decode_value_time; + materialization_time += col_statistics.materialization_time; + hybrid_selection_batches += col_statistics.hybrid_selection_batches; + hybrid_selection_ranges += col_statistics.hybrid_selection_ranges; + hybrid_selection_null_fallback_batches += + col_statistics.hybrid_selection_null_fallback_batches; + decode_dict_time += col_statistics.decode_dict_time; + decode_level_time += col_statistics.decode_level_time; + decode_null_map_time += col_statistics.decode_null_map_time; + skip_page_header_num += col_statistics.skip_page_header_num; + parse_page_header_num += col_statistics.parse_page_header_num; + read_page_header_time += col_statistics.read_page_header_time; + page_read_counter += col_statistics.page_read_counter; + leaf_page_read_counters.insert(leaf_page_read_counters.end(), + col_statistics.leaf_page_read_counters.begin(), + col_statistics.leaf_page_read_counters.end()); + page_cache_write_counter += col_statistics.page_cache_write_counter; + page_cache_compressed_write_counter += + col_statistics.page_cache_compressed_write_counter; + page_cache_decompressed_write_counter += + col_statistics.page_cache_decompressed_write_counter; + page_cache_hit_counter += col_statistics.page_cache_hit_counter; + page_cache_missing_counter += col_statistics.page_cache_missing_counter; + page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter; + page_cache_decompressed_hit_counter += + col_statistics.page_cache_decompressed_hit_counter; + } + }; + + ColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} + virtual ~ColumnReader() = default; + virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter, + int64_t real_column_size = -1) = 0; + + // Evaluate a predicate scalar directly from a supported fixed-width page encoding. The default + // is a non-consuming fallback for nested and synthetic readers. + virtual Status read_fixed_width_filter(const VExprSPtrs&, int, FilterMap&, size_t, IColumn*, + IColumn::Filter* row_filter, size_t* read_rows, + bool* eof, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + return Status::OK(); + } + + // Consume a nested batch while retaining only definition/repetition levels. This is used when + // schema evolution makes every projected STRUCT child synthetic: the parent still needs one + // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. + virtual Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) = 0; + + virtual Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) { + throw Exception( + Status::FatalError("Method materialize_dictionary_values is not supported")); + } + virtual Result dictionary_values(const DataTypePtr& target_type) { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } + + static Status create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection = false, + const std::set& column_ids = {}, + const std::set& filter_column_ids = {}, + const std::string& page_cache_file_key = {}, + const ParquetReaderCompat& compat = {}, bool enable_strict_mode = false); + virtual const std::vector& get_rep_level() const = 0; + virtual const std::vector& get_def_level() const = 0; + virtual ColumnStatistics column_statistics() = 0; + virtual void close() = 0; + + // A repeated parent can expand one logical-row batch into millions of leaf values. Keep + // ordinary batch scratch for reuse, but let the top-level adapter release exceptional + // high-water allocations after every parent offset/null-map consumer has finished. + virtual void release_batch_scratch(size_t max_retained_bytes) = 0; + + virtual void reset_filter_map_index() = 0; + + NativeFieldSchema* get_field_schema() const { return _field_schema; } + void set_column_in_nested() { _in_nested = true; } + +protected: + void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; + + NativeFieldSchema* _field_schema = nullptr; + const RowRanges& _row_ranges; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _current_row_index = 0; + int64_t _decode_null_map_time = 0; + + size_t _filter_map_index = 0; + std::set _filter_column_ids; + + // _in_nested: column in struct/map/array + // IN_COLLECTION : column in map/array + bool _in_nested = false; +}; + +template +class ScalarColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ScalarColumnReader) +public: + ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} + ~ScalarColumnReader() override { close(); } + Status init(io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, + RuntimeState* state, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_fixed_width_filter(const VExprSPtrs& conjuncts, int column_id, + FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, + size_t* read_rows, bool* eof, bool* used_filter) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) override; + Result dictionary_values(const DataTypePtr& target_type) override; + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time); + } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override; + +#ifdef BE_TEST + void reserve_batch_scratch_for_test(size_t elements); + size_t retained_batch_scratch_bytes_for_test() const; +#endif + + void reset_filter_map_index() override { + _filter_map_index = 0; // nested + } + +private: + tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index = nullptr; + std::unique_ptr _stream_reader; + std::unique_ptr> _chunk_reader; + // rep def levels buffer. + std::vector _rep_levels; + std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + FilterMap* nested_filter_map) { + DORIS_CHECK(nested_filter_map != nullptr); + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + return nested_filter_map->init(nested_filter_map_data.data(), nested_filter_map_data.size(), + false); + } + + DataTypeSerDeSPtr _serde; + const IDataType* _serde_type = nullptr; + ParquetDecodeContext _decode_context; + ParquetMaterializationState _materialization_state; + bool _dictionary_index_only = false; + // Normal-size batch scratch is retained by the persistent leaf reader. Oversized allocations + // are released only after the top-level parent has consumed this leaf's level plan. + std::vector _null_run_lengths; + std::unordered_set _ancestor_null_indices; + std::vector _nested_filter_map_data; + NullMap _fixed_width_predicate_nulls; + IColumn::Filter _fixed_width_predicate_matches; + FilterMap _nested_filter_map; + ColumnSelectVector _select_vector; + uint8_t _oversized_scratch_idle_batches = 0; + + size_t retained_batch_scratch_bytes() const; + size_t active_batch_scratch_bytes() const; + + Status _skip_values(size_t num_values); + Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, bool is_dict_filter); + Status _read_fixed_width_filter_values(size_t num_values, const VExprSPtrs& conjuncts, + int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter); + Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter); + Status _try_load_dict_page(bool* loaded, bool* has_dict); +}; + +class ArrayColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ArrayColumnReader) +public: + ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~ArrayColumnReader() override { close(); } + Status init(std::unique_ptr element_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override { + return _element_reader->read_column_levels(filter_map, batch_size, read_rows, eof); + } + const std::vector& get_rep_level() const override { + return _element_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _element_reader->get_def_level(); + } + ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _element_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } + +private: + std::unique_ptr _element_reader; +}; + +class MapColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(MapColumnReader) +public: + MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~MapColumnReader() override { close(); } + + Status init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + return _key_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _key_reader->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics kst = _key_reader->column_statistics(); + ColumnStatistics vst = _value_reader->column_statistics(); + kst.merge(vst); + return kst; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _key_reader->release_batch_scratch(max_retained_bytes); + _value_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { + _key_reader->reset_filter_map_index(); + _value_reader->reset_filter_map_index(); + } + +private: + std::unique_ptr _key_reader; + std::unique_ptr _value_reader; +}; + +class StructColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(StructColumnReader) +public: + StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~StructColumnReader() override { close(); } + + Status init(std::unordered_map>&& child_readers, + NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + if (!_read_column_names.empty()) { + // can't use _child_readers[*_read_column_names.begin()] + // because the operator[] of std::unordered_map is not const :( + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column, + * that is `_read_column_names.front()`. + */ + return _child_readers.find(_read_column_names.front())->second->get_rep_level(); + } + return _child_readers.begin()->second->get_rep_level(); + } + + const std::vector& get_def_level() const override { + if (!_read_column_names.empty()) { + return _child_readers.find(_read_column_names.front())->second->get_def_level(); + } + return _child_readers.begin()->second->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics st; + for (const auto& column_name : _read_column_names) { + auto reader = _child_readers.find(column_name); + if (reader != _child_readers.end()) { + ColumnStatistics cst = reader->second->column_statistics(); + st.merge(cst); + } + } + return st; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + for (const auto& reader : _child_readers) { + reader.second->release_batch_scratch(max_retained_bytes); + } + } + + void reset_filter_map_index() override { + for (const auto& reader : _child_readers) { + reader.second->reset_filter_map_index(); + } + } + +private: + std::unordered_map> _child_readers; + std::vector _read_column_names; + //Need to use vector instead of set,see `get_rep_level()` for the reason. +}; + +// A special reader that skips actual reading but provides empty data with correct structure +// This is used when a column is not needed but its structure is required (e.g., for map keys) +class SkipReadingReader : public ColumnReader { +public: + SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx, NativeFieldSchema* field_schema) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { + _field_schema = field_schema; // Use inherited member from base class + VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " + << _field_schema->name; + } + + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override { + VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " + << _field_schema->name << ", batch_size: " << batch_size; + DCHECK(real_column_size >= 0); // real_column_size for filtered column size. + + // Simulate reading without actually reading data + // Fill with default/null values based on column type + doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column = doris_column->assert_mutable(); + + if (real_column_size > 0) { + if (is_column_nullable(*doris_column)) { + auto* nullable_column = static_cast(data_column.get()); + nullable_column->insert_many_defaults(real_column_size); + } else { + // For non-nullable columns, insert appropriate default values + for (size_t i = 0; i < real_column_size; ++i) { + data_column->insert_default(); + } + } + } + + *read_rows = batch_size; // Indicate we "read" batch_size rows + *eof = false; // We can always provide more empty data + + VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size + << " default values for field: " << _field_schema->name; + + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t*, bool*) override { + return Status::InternalError("Skip reader cannot provide Parquet levels for field {}", + _field_schema->name); + } + + static std::unique_ptr create_unique(const RowRanges& row_ranges, + size_t total_rows, cctz::time_zone* ctz, + io::IOContext* io_ctx, + NativeFieldSchema* field_schema) { + return std::make_unique(row_ranges, total_rows, ctz, io_ctx, + field_schema); + } + + // These methods should not be called for SkipReadingReader + // If they are called, it indicates a logic error in the code + const std::vector& get_rep_level() const override { + LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + const std::vector& get_def_level() const override { + LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + // Implement required pure virtual methods from base class + ColumnStatistics column_statistics() override { + return ColumnStatistics(); // Return empty statistics + } + + void close() override { + // Nothing to close for skip reading + } + + void release_batch_scratch(size_t) override {} + + void reset_filter_map_index() override { _filter_map_index = 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.cpp b/be/src/format_v2/parquet/reader/native/common.cpp new file mode 100644 index 00000000000000..f048d2b878ebee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/common.h" + +#include + +#include "core/types.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet::native { + +Status FilterMap::init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all) { + _filter_all = filter_all; + _filter_map_data = filter_map_data; + _filter_map_size = filter_map_size; + if (filter_all) { + _has_filter = true; + _filter_ratio = 1; + } else if (filter_map_data == nullptr) { + _has_filter = false; + _filter_ratio = 0; + } else { + const size_t filter_count = simd::count_zero_num( + reinterpret_cast(filter_map_data), filter_map_size); + if (filter_count == filter_map_size) { + _has_filter = true; + _filter_all = true; + _filter_ratio = 1; + } else if (filter_count > 0 && filter_map_size > 0) { + _has_filter = true; + _filter_ratio = static_cast(filter_count) / filter_map_size; + } else { + _has_filter = false; + _filter_ratio = 0; + } + } + return Status::OK(); +} + +bool FilterMap::can_filter_all(size_t remaining_num_values, size_t filter_map_index) { + if (!_has_filter) { + return false; + } + if (_filter_all) { + DCHECK_LE(remaining_num_values + filter_map_index, _filter_map_size); + return true; + } + if (remaining_num_values + filter_map_index > _filter_map_size) { + return false; + } + return simd::count_zero_num( + reinterpret_cast(_filter_map_data + filter_map_index), + remaining_num_values) == remaining_num_values; +} + +Status FilterMap::generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index) const { + if (!has_filter() || filter_all()) { + return Status::InternalError( + "Native FilterMap requires a partial filter: has_filter={}, filter_all={}", + has_filter(), filter_all()); + } + if (rep_levels.empty()) { + return Status::OK(); + } + + nested_filter_map_data.resize(rep_levels.size()); + size_t current_row = current_row_ptr != nullptr ? *current_row_ptr : 0; + for (size_t i = start_index; i < rep_levels.size(); ++i) { + if (i != start_index && rep_levels[i] == 0) { + ++current_row; + if (current_row >= _filter_map_size) { + return Status::InvalidArgument("Nested filter row {} exceeds filter map size {}", + current_row, _filter_map_size); + } + } + nested_filter_map_data[i] = _filter_map_data[current_row]; + } + if (current_row_ptr != nullptr) { + *current_row_ptr = current_row; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR( + new_filter->init(nested_filter_map_data.data(), nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + return Status::OK(); +} + +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (_has_filter) { + _data_map.resize(num_values); + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t source_index = 0; + size_t valid_count = 0; + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + source_index, filter_map->filter_map_size()); + if (skipped_indices != nullptr && + skipped_indices->contains(filter_map_index + source_index)) { + ++source_index; + continue; + } + if (filter_map->filter_map_data()[filter_map_index + source_index] != 0) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + ++num_read; + } + ++valid_count; + ++source_index; + } + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_read); + if (_num_nulls == 0) { + memset(null_map->data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_index, 1, num_read); + } else { + for (size_t i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + (*null_map)[null_map_index++] = static_cast(false); + } else if (_data_map[i] == NULL_DATA) { + (*null_map)[null_map_index++] = static_cast(true); + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_values); + for (const auto run_length : run_length_null_map) { + memset(null_map->data() + null_map_index, is_null ? 1 : 0, run_length); + null_map_index += run_length; + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } else { + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.h b/be/src/format_v2/parquet/reader/native/common.h new file mode 100644 index 00000000000000..4ad204da2ff30d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet::native { + +// These cursor types belong to the native v2 decoder so its execution path never reaches into +// the legacy Parquet reader for filtering or level materialization. +using level_t = int16_t; + +class FilterMap { +public: + Status init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all); + + Status generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index = 0) const; + + const uint8_t* filter_map_data() const { return _filter_map_data; } + size_t filter_map_size() const { return _filter_map_size; } + bool has_filter() const { return _has_filter; } + bool filter_all() const { return _filter_all; } + double filter_ratio() const { return _has_filter ? _filter_ratio : 0; } + + bool can_filter_all(size_t remaining_num_values, size_t filter_map_index); + +private: + bool _filter_all = false; + bool _has_filter = false; + const uint8_t* _filter_map_data = nullptr; + size_t _filter_map_size = 0; + double _filter_ratio = 0; +}; + +class ColumnSelectVector { +public: + enum DataReadType : uint8_t { CONTENT = 0, NULL_DATA, FILTERED_CONTENT, FILTERED_NULL }; + + Status init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices = nullptr); + + size_t num_values() const { return _num_values; } + size_t num_nulls() const { return _num_nulls; } + size_t num_filtered() const { return _num_filtered; } + bool has_filter() const { return _has_filter; } + + template + size_t get_next_run(DataReadType* data_read_type) { + DCHECK_EQ(_has_filter, has_filter); + if constexpr (has_filter) { + if (_read_index == _num_values) { + return 0; + } + const DataReadType type = _data_map[_read_index++]; + size_t run_length = 1; + while (_read_index < _num_values && _data_map[_read_index] == type) { + ++run_length; + ++_read_index; + } + *data_read_type = type; + return run_length; + } + + size_t run_length = 0; + while (run_length == 0) { + if (_read_index == _run_length_null_map->size()) { + return 0; + } + *data_read_type = _read_index % 2 == 0 ? CONTENT : NULL_DATA; + run_length = (*_run_length_null_map)[_read_index++]; + } + return run_length; + } + +private: + std::vector _data_map; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp new file mode 100644 index 00000000000000..5f6a978317bde7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/decoder.h" + +#include +#include + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_dict_decoder.hpp" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { +namespace { +Status unsupported_type(tparquet::Type::type type, tparquet::Encoding::type encoding) { + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); +} + +Status create_plain_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::PLAIN); + } +} + +Status create_dictionary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Boolean type cannot have a dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT96: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FLOAT: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::DOUBLE: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique>(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::RLE_DICTIONARY); + } +} + +Status create_delta_binary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } +} + +Status create_byte_stream_split_decoder(tparquet::Type::type type, + std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::BYTE_STREAM_SPLIT); + } +} +} // namespace + +Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return create_plain_decoder(type, decoder); + case tparquet::Encoding::RLE_DICTIONARY: + return create_dictionary_decoder(type, decoder); + case tparquet::Encoding::RLE: + if (type != tparquet::Type::BOOLEAN) { + return unsupported_type(type, encoding); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + return create_delta_binary_decoder(type, decoder); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY && type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) { + return Status::InternalError( + "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY) { + return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return create_byte_stream_split_decoder(type, decoder); + default: + return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", + tparquet::to_string(encoding), tparquet::to_string(type)); + } +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h new file mode 100644 index 00000000000000..379176d5e459e0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __AVX2__ +#include +#endif + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/types.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { + +inline bool dictionary_indices_in_bounds(const uint32_t* indices, size_t count, + size_t dictionary_size) { + if (count == 0) { + return true; + } + if (dictionary_size == 0) { + return false; + } + uint32_t max_index = 0; + size_t row = 0; +#ifdef __AVX2__ + __m256i vector_max = _mm256_setzero_si256(); + for (; row + 8 <= count; row += 8) { + const auto values = _mm256_loadu_si256(reinterpret_cast(indices + row)); + vector_max = _mm256_max_epu32(vector_max, values); + } + alignas(32) uint32_t lanes[8]; + _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), vector_max); + for (const auto lane : lanes) { + max_index = std::max(max_index, lane); + } +#endif + for (; row < count; ++row) { + max_index = std::max(max_index, indices[row]); + } + return static_cast(max_index) < dictionary_size; +} + +class Decoder : public ParquetDecodeSource { +public: + Decoder() = default; + virtual ~Decoder() = default; + + static Status get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder); + + // The type with fix length + void set_type_length(int32_t type_length) { _type_length = type_length; } + + // Set the data to be decoded + virtual Status set_data(Slice* data) { + _data = data; + _offset = 0; + return Status::OK(); + } + + // Page headers provide an upper bound before encoding-specific headers are trusted. + virtual void set_expected_values(size_t expected_values) { _expected_values = expected_values; } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("Binary values are not supported by this Parquet decoder"); + } + + Status skip_values(size_t num_values) override = 0; + + virtual void release_scratch(size_t max_retained_bytes) {} + virtual size_t retained_scratch_bytes() const { return 0; } + virtual size_t active_scratch_bytes() const { return 0; } + + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + return Status::NotSupported("set_dict is not supported"); + } + +protected: + template + static void release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + if (values->capacity() * sizeof(T) > max_retained_bytes) { + std::vector().swap(*values); + } + } + + int32_t _type_length = -1; + Slice* _data = nullptr; + // Page offsets are host-sized so checked advances cannot wrap at the 4 GiB boundary. + size_t _offset = 0; + size_t _expected_values = std::numeric_limits::max(); +}; + +class BaseDictDecoder : public Decoder { +public: + BaseDictDecoder() = default; + ~BaseDictDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + if (UNLIKELY(data == nullptr || data->size == 0)) { + return Status::Corruption("Parquet dictionary index stream is empty"); + } + _data = data; + _offset = 0; + uint8_t bit_width = *data->data; + // Dictionary indices are uint32_t; wider external widths make repeated runs overwrite the + // decoder's four-byte state before any dictionary-bound check can run. + if (UNLIKELY(bit_width > 32)) { + return Status::Corruption("Parquet dictionary index bit width {} exceeds 32", + bit_width); + } + _index_batch_decoder = std::make_unique>( + reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, + bit_width); + return Status::OK(); + } + + bool has_dictionary() const override { return true; } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + indices->resize(num_values); + const auto decoded = + _index_batch_decoder->GetBatch(indices->data(), cast_set(num_values)); + if (UNLIKELY(decoded != num_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data(), num_values, + num_dictionary_values))) { + // The SIMD common path only computes a bound; recover the exact corrupt row for the + // diagnostic after the batch has already been proven invalid. + for (size_t row = 0; row < num_values; ++row) { + if ((*indices)[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + + Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + const size_t num_dictionary_values = dictionary_size(); + indices->resize(selection.selected_values); + size_t cursor = 0; + size_t output = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + const auto decoded = _index_batch_decoder->GetBatch(indices->data() + output, + cast_set(range.count)); + if (UNLIKELY(decoded != range.count)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data() + output, range.count, + num_dictionary_values))) { + for (size_t row = 0; row < range.count; ++row) { + if ((*indices)[output + row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[output + row], range.first + row, num_dictionary_values); + } + } + output += range.count; + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + RETURN_IF_ERROR(_decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values)); + DORIS_CHECK_EQ(output, selection.selected_values); + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); + } + + Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { + const size_t num_dictionary_values = dictionary_size(); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + RETURN_IF_ERROR(_decode_dictionary_values(range.count, range.first, + num_dictionary_values, consumer)); + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + return _decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_skip_indices, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _skip_indices.capacity() * sizeof(uint32_t); + } + size_t active_scratch_bytes() const override { return _skip_indices.size() * sizeof(uint32_t); } + +protected: + Status skip_values(size_t num_values) override { + return _decode_and_validate_skipped(num_values, 0, dictionary_size()); + } + + Status _decode_and_validate_skipped(size_t num_values, size_t row_offset, + size_t num_dictionary_values) { + constexpr size_t kSkipBatchSize = 4096; + // Skipped dictionary ids are still external input and must be bounds-checked, but keeping + // only one bounded gap buffer avoids the page-sized scratch used by sparse selections. + _skip_indices.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped_values = 0; + while (skipped_values < num_values) { + const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + static_cast(batch_size)); + if (UNLIKELY(skipped != batch_size)) { + return Status::IOError( + "Can't skip enough Parquet dictionary indices at row {}: {} of {}", + row_offset + skipped_values, skipped, batch_size); + } + // Filter gaps may be huge RLE runs; validate them in bounded SIMD-sized batches. + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch_size, + num_dictionary_values))) { + for (size_t row = 0; row < batch_size; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary " + "size {}", + _skip_indices[row], row_offset + skipped_values + row, + num_dictionary_values); + } + } + skipped_values += batch_size; + } + return Status::OK(); + } + + Status _decode_dictionary_values(size_t num_values, size_t row_offset, + size_t num_dictionary_values, + ParquetDictionaryValueConsumer& consumer) { + constexpr size_t kLiteralBatchSize = 1024; + size_t decoded_values = 0; + while (decoded_values < num_values) { + const int32_t repeats = _index_batch_decoder->NextNumRepeats(); + if (repeats > 0) { + const size_t run = std::min(repeats, num_values - decoded_values); + const uint32_t index = + _index_batch_decoder->GetRepeatedValue(cast_set(run)); + if (UNLIKELY(static_cast(index) >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + index, row_offset + decoded_values, num_dictionary_values); + } + RETURN_IF_ERROR(consumer.consume_repeated(index, run)); + decoded_values += run; + continue; + } + + const int32_t literals = _index_batch_decoder->NextNumLiterals(); + if (UNLIKELY(literals == 0)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t batch = std::min({static_cast(literals), + num_values - decoded_values, kLiteralBatchSize}); + _skip_indices.resize(batch); + if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), + _skip_indices.data()))) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch, + num_dictionary_values))) { + for (size_t row = 0; row < batch; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row_offset + decoded_values + row, + num_dictionary_values); + } + } + RETURN_IF_ERROR(consumer.consume_indices(_skip_indices.data(), batch)); + decoded_values += batch; + } + return Status::OK(); + } + + // For dictionary encoding + DorisUniqueBufferPtr _dict; + std::unique_ptr> _index_batch_decoder; + std::vector _skip_indices; + uint64_t _dictionary_generation = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp new file mode 100644 index 00000000000000..c94aead8caa6b6 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" + +namespace doris::format::parquet::native { +Status DeltaLengthByteArrayDecoder::_init_lengths() { + auto length_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_len_decoder.set_bit_reader(length_reader)); + const uint32_t num_lengths = _len_decoder.valid_values_count(); + + DeltaBitPackDecoder length_locator; + length_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(length_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector lengths(std::min(num_lengths, kLocateBatchSize)); + size_t located = 0; + int64_t payload_size = 0; + while (located < num_lengths) { + const size_t batch_size = std::min(num_lengths - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR( + length_locator.decode(lengths.data(), static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + for (size_t i = 0; i < batch_size; ++i) { + if (UNLIKELY(lengths[i] < 0) || + common::add_overflow(payload_size, static_cast(lengths[i]), + payload_size)) { + return Status::Corruption("Invalid Parquet delta byte-array length"); + } + } + located += batch_size; + } + // The locator leaves the shared reader at the first payload byte without retaining every + // length; the independent decoder supplies bounded batches during materialization or skip. + if (UNLIKELY(payload_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + payload_size, _bit_reader->bytes_left()); + } + _num_valid_values = num_lengths; + return Status::OK(); +} + +Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, + int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + + int64_t data_size = 0; + _buffered_length.resize(max_values); + uint32_t lengths_decoded = 0; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), static_cast(max_values), + &lengths_decoded)); + if (UNLIKELY(lengths_decoded != max_values)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + const int32_t* length_ptr = _buffered_length.data(); + for (int i = 0; i < max_values; ++i) { + int32_t len = length_ptr[i]; + if (len < 0) [[unlikely]] { + return Status::InvalidArgument("Negative string delta length"); + } + buffer[i].size = len; + if (common::add_overflow(data_size, static_cast(len), data_size)) { + return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); + } + } + // Every declared byte must exist in this page. Check before resize so a tiny malformed stream + // cannot reserve memory based only on attacker-controlled decoded lengths. + if (UNLIKELY(data_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + data_size, _bit_reader->bytes_left()); + } + _buffered_data.resize(data_size); + char* data_ptr = _buffered_data.data(); + for (int64_t j = 0; j < data_size; j++) { + if (!_bit_reader->GetValue(8, data_ptr + j)) { + return Status::IOError("Get length bytes EOF"); + } + } + + for (int i = 0; i < max_values; ++i) { + buffer[i].data = data_ptr; + data_ptr += buffer[i].size; + } + // this->num_values_ -= max_values; + _num_valid_values -= max_values; + *out_num_values = max_values; + return Status::OK(); +} + +Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = max_values; + return Status::OK(); + } + + int suffix_read; + RETURN_IF_ERROR(_suffix_decoder.decode(buffer, max_values, &suffix_read)); + if (suffix_read != max_values) [[unlikely]] { + return Status::IOError("Read {}, expecting {} from suffix decoder", + std::to_string(suffix_read), std::to_string(max_values)); + } + + int64_t data_size = 0; + _buffered_prefix_length.resize(max_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), static_cast(max_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != max_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + const int32_t* prefix_len_ptr = _buffered_prefix_length.data(); + size_t preceding_value_size = _last_value.size(); + for (int i = 0; i < max_values; ++i) { + if (prefix_len_ptr[i] < 0) [[unlikely]] { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(prefix_len_ptr[i]); + if (prefix_size > preceding_value_size) [[unlikely]] { + // Prefixes form a dependency chain, so validate each one before aggregate allocation. + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (common::add_overflow(prefix_size, buffer[i].size, reconstructed_size) || + reconstructed_size > static_cast(std::numeric_limits::max()) || + common::add_overflow(data_size, static_cast(reconstructed_size), data_size)) + [[unlikely]] { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + preceding_value_size = reconstructed_size; + } + _buffered_data.resize(data_size); + + std::string_view prefix {_last_value}; + + char* data_ptr = _buffered_data.data(); + for (int i = 0; i < max_values; ++i) { + memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); + // buffer[i] currently points to the string suffix + memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); + buffer[i].data = data_ptr; + buffer[i].size += prefix_len_ptr[i]; + data_ptr += buffer[i].size; + prefix = std::string_view {buffer[i].data, buffer[i].size}; + } + _num_valid_values -= max_values; + _last_value = std::string {prefix}; + + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + *out_num_values = max_values; + return Status::OK(); +} +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h new file mode 100644 index 00000000000000..531cb7704075b5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -0,0 +1,812 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "exec/common/arithmetic_overflow.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +namespace detail { + +inline Status checked_delta_padding_bits(uint32_t bit_width, uint32_t remaining_values, + int64_t* padding_bits) { + DORIS_CHECK(padding_bits != nullptr); + const uint64_t widened_bits = static_cast(bit_width) * remaining_values; + if (UNLIKELY(widened_bits > static_cast(std::numeric_limits::max()))) { + return Status::Corruption("Parquet delta miniblock padding is too large"); + } + *padding_bits = static_cast(widened_bits); + return Status::OK(); +} + +} // namespace detail + +class DeltaDecoder : public Decoder { +public: + DeltaDecoder() = default; + ~DeltaDecoder() override = default; +}; + +/** + * Format + * [header] [block 1] [block 2] ... [block N] + * Header + * [block size] [_mini_blocks_per_block] [_total_value_count] [first value] + * Block + * [min delta] [list of bitwidths of the mini blocks] [miniblocks] + */ +template +class DeltaBitPackDecoder final : public DeltaDecoder { +public: + using UT = std::make_unsigned_t; + + DeltaBitPackDecoder() = default; + ~DeltaBitPackDecoder() override = default; + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), static_cast(batch_size), + &num_valid_values)); + // Skips retain exact validation without allocating in proportion to a sparse gap. + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + _values.resize(num_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", num_values, + decoded_count); + } + return consumer.consume(reinterpret_cast(_values.data()), num_values, + sizeof(T)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + _values.resize(selection.total_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", + selection.total_values, decoded_count); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, range.count * sizeof(T)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(reinterpret_cast(_values.data()), output, + sizeof(T)); + } + + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + uint32_t valid_values_count() { + // _total_value_count in header ignores of null values + return _total_values_remaining; + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + // The bit-width table drives later miniblock transitions, so it is reclaimable only after + // the page is exhausted; _values contains completed batch output and is always disposable. + if (_total_values_remaining == 0) { + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(T) + _delta_bit_widths.size() * sizeof(uint8_t); + } + + Status set_data(Slice* slice) override { + _bit_reader.reset( + new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); + RETURN_IF_ERROR(_init_header()); + _data = slice; + _offset = 0; + return Status::OK(); + } + + // Set BitReader which is already initialized by DeltaLengthByteArrayDecoder or + // DeltaByteArrayDecoder + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_header()); + return Status::OK(); + } + +private: + static constexpr int kMaxDeltaBitWidth = static_cast(sizeof(T) * 8); + Status _init_header(); + Status _init_block(); + Status _init_mini_block(int bit_width); + Status _get_internal(T* buffer, uint32_t max_values, uint32_t* out_num_values); + + std::vector _values; + + std::shared_ptr _bit_reader; + uint32_t _values_per_block; + uint32_t _mini_blocks_per_block; + uint32_t _values_per_mini_block; + uint32_t _total_value_count; + + T _min_delta; + T _last_value; + + uint32_t _mini_block_idx; + std::vector _delta_bit_widths; + int _delta_bit_width; + // If the page doesn't contain any block, `_block_initialized` will + // always be false. Otherwise, it will be true when first block initialized. + bool _block_initialized; + + uint32_t _total_values_remaining; + // Remaining values in current mini block. If the current block is the last mini block, + // _values_remaining_current_mini_block may greater than _total_values_remaining. + uint32_t _values_remaining_current_mini_block; +}; + +class DeltaLengthByteArrayDecoder final : public DeltaDecoder { +public: + explicit DeltaLengthByteArrayDecoder() + : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _len_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), static_cast(batch_size), &num_valid_values)); + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError( + "Expected to skip {} Parquet delta-length values, skipped {}", num_values, + skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + num_values, decoded_count); + } + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _values.resize(selection.total_values); + int decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + selection.total_values, decoded_count); + } + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + int valid_values_count() const { return _num_valid_values; } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_buffered_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _len_decoder.release_scratch(max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _buffered_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _buffered_length.size() * sizeof(int32_t) + _buffered_data.size() * sizeof(char) + + _len_decoder.active_scratch_bytes(); + } + + Status set_data(Slice* slice) override { + if (slice == nullptr || slice->size == 0) { + // Reused decoders must never retain lengths or payload pointers from the prior page. + _bit_reader.reset(); + _data = nullptr; + _offset = 0; + _num_valid_values = 0; + _buffered_length.clear(); + _buffered_data.clear(); + return Status::Corruption("Parquet delta-length page is empty"); + } + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + _data = slice; + _offset = 0; + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + +private: + Status _init_lengths(); + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _len_decoder; + + int _num_valid_values; + std::vector _buffered_length; + std::vector _buffered_data; +}; + +class DeltaByteArrayDecoder : public DeltaDecoder { +public: + explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _prefix_len_decoder.set_expected_values(expected_values); + _suffix_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + RETURN_IF_ERROR(_skip_slices(batch_size)); + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _selected_binary_values.clear(); + _selected_value_sizes.clear(); + _selected_value_sizes.reserve(selection.selected_values); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + for (const auto& value : _values) { + if (UNLIKELY(value.size > std::numeric_limits::max() - + _selected_binary_values.size())) { + return Status::IOError( + "Parquet delta-byte-array selection byte size overflows"); + } + if (value.size > 0) { + _selected_binary_values.insert(_selected_binary_values.end(), value.data, + value.data + value.size); + } + _selected_value_sizes.push_back(value.size); + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + + _string_refs.resize(selection.selected_values); + size_t byte_offset = 0; + for (size_t output = 0; output < _selected_value_sizes.size(); ++output) { + const size_t value_size = _selected_value_sizes[output]; + const char* value_data = + value_size == 0 ? nullptr : _selected_binary_values.data() + byte_offset; + _string_refs[output] = StringRef(value_data, value_size); + byte_offset += value_size; + } + DORIS_CHECK_EQ(_selected_value_sizes.size(), selection.selected_values); + DORIS_CHECK_EQ(byte_offset, _selected_binary_values.size()); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + const size_t byte_size = num_values * static_cast(_type_length); + _fixed_values.resize(byte_size); + for (size_t row = 0; row < num_values; ++row) { + memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); + } + return consumer.consume(_fixed_values.data(), num_values, + static_cast(_type_length)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.selected_values > + std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet delta-byte-array selection byte size overflows"); + } + _fixed_values.resize(selection.selected_values * value_width); + size_t output = 0; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + // Skips validate widths too, so filtering cannot hide a malformed value after the + // decoder cursor has advanced past it. + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + for (const auto& value : _values) { + memcpy(_fixed_values.data() + output * value_width, value.data, value_width); + ++output; + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_fixed_values.data(), output, value_width); + } + + Status set_data(Slice* slice) override { + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + auto prefix_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(prefix_reader)); + + // get the number of encoded prefix lengths + int num_prefix = _prefix_len_decoder.valid_values_count(); + DeltaBitPackDecoder prefix_locator; + prefix_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(prefix_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector ignored_prefixes( + std::min(static_cast(num_prefix), kLocateBatchSize)); + size_t located = 0; + while (located < static_cast(num_prefix)) { + const size_t batch_size = + std::min(static_cast(num_prefix) - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR(prefix_locator.decode(ignored_prefixes.data(), + static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + located += batch_size; + } + _num_valid_values = num_prefix; + + // at this time, the decoder_ will be at the start of the encoded suffix data. + RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + if (UNLIKELY(_suffix_decoder.valid_values_count() != num_prefix)) { + return Status::Corruption("Parquet delta prefix and suffix counts differ"); + } + + // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set + // to _last_value_in_previous_page when decoding a new page(except the first page) + _last_value = ""; + return Status::OK(); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_fixed_values, max_retained_bytes); + release_vector_if_oversized(&_selected_binary_values, max_retained_bytes); + release_vector_if_oversized(&_selected_value_sizes, max_retained_bytes); + release_vector_if_oversized(&_buffered_prefix_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _prefix_len_decoder.release_scratch(max_retained_bytes); + _suffix_decoder.release_scratch(max_retained_bytes); + // Prefix reconstruction crosses batch boundaries, so the previous value remains semantic + // decoder state until every value in the current page has been consumed. + if (_num_valid_values == 0) { + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _fixed_values.capacity() * sizeof(uint8_t) + + _selected_binary_values.capacity() * sizeof(char) + + _selected_value_sizes.capacity() * sizeof(size_t) + + _buffered_prefix_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _last_value.capacity() + + _last_value_in_previous_page.capacity() + + _prefix_len_decoder.retained_scratch_bytes() + + _suffix_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _fixed_values.size() * sizeof(uint8_t) + + _selected_binary_values.size() * sizeof(char) + + _selected_value_sizes.size() * sizeof(size_t) + + _buffered_prefix_length.size() * sizeof(int32_t) + + _buffered_data.size() * sizeof(char) + _last_value.size() + + _last_value_in_previous_page.size() + _prefix_len_decoder.active_scratch_bytes() + + _suffix_decoder.active_scratch_bytes(); + } + +private: + size_t _bounded_reconstruction_batch_size(size_t remaining_values) const { + constexpr size_t kReconstructionByteBudget = 4UL << 20; + constexpr size_t kMaxReconstructionValues = 4096; + // Prefix expansion, rather than encoded page bytes, determines reconstruction memory. A + // count-based chunk could turn one long value into gigabytes of temporary copies, so first + // learn its width and then cap subsequent chunks by a byte budget. + if (_last_value.empty()) { + return 1; + } + const size_t budgeted_values = + std::max(1, kReconstructionByteBudget / _last_value.size()); + return std::min({remaining_values, kMaxReconstructionValues, budgeted_values}); + } + + Status _skip_slices(size_t num_values) { + _values.resize(num_values); + int suffixes_decoded = 0; + RETURN_IF_ERROR(_suffix_decoder.decode(_values.data(), cast_set(num_values), + &suffixes_decoded)); + if (UNLIKELY(suffixes_decoded != num_values)) { + return Status::IOError("Expected to skip {} Parquet delta suffixes, skipped {}", + num_values, suffixes_decoded); + } + _buffered_prefix_length.resize(num_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), cast_set(num_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != num_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + + // Only the preceding value is semantic decoder state. Rebuilding every skipped value into + // one aggregate buffer would amplify compact full-prefix references into page_size * rows. + for (size_t row = 0; row < num_values; ++row) { + const int32_t encoded_prefix_size = _buffered_prefix_length[row]; + if (UNLIKELY(encoded_prefix_size < 0)) { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(encoded_prefix_size); + if (UNLIKELY(prefix_size > _last_value.size())) { + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (UNLIKELY( + common::add_overflow(prefix_size, _values[row].size, reconstructed_size))) { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + if (_type_length > 0 && + UNLIKELY(reconstructed_size != static_cast(_type_length))) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + reconstructed_size, _type_length); + } + // resize() preserves the prefix already stored in _last_value, so only the suffix + // needs copying and repeated full-prefix values allocate no additional payload buffer. + _last_value.resize(reconstructed_size); + if (_values[row].size > 0) { + memcpy(_last_value.data() + prefix_size, _values[row].data, _values[row].size); + } + } + _num_valid_values -= cast_set(num_values); + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + return Status::OK(); + } + + Status _validate_fixed_width_values() const { + if (_type_length <= 0) { + return Status::OK(); + } + const size_t value_width = static_cast(_type_length); + for (const auto& value : _values) { + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + } + return Status::OK(); + } + + Status _decode_slices(size_t num_values) { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-byte-array values, decoded {}", + num_values, decoded_count); + } + return Status::OK(); + } + + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::vector _fixed_values; + std::vector _selected_binary_values; + std::vector _selected_value_sizes; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _prefix_len_decoder; + DeltaLengthByteArrayDecoder _suffix_decoder; + std::string _last_value; + // string buffer for last value in previous page + std::string _last_value_in_previous_page; + int _num_valid_values; + std::vector _buffered_prefix_length; + std::vector _buffered_data; +}; +} // namespace doris::format::parquet::native + +namespace doris::format::parquet::native { + +template +Status DeltaBitPackDecoder::_init_header() { + if (!_bit_reader->GetVlqInt(&_values_per_block) || + !_bit_reader->GetVlqInt(&_mini_blocks_per_block) || + !_bit_reader->GetVlqInt(&_total_value_count) || + !_bit_reader->GetZigZagVlqInt(&_last_value)) { + return Status::IOError("Init header eof"); + } + if (_values_per_block == 0) { + return Status::InvalidArgument("Cannot have zero value per block"); + } + if (_values_per_block % 128 != 0) { + return Status::InvalidArgument( + "the number of values in a block must be multiple of 128, but it's " + + std::to_string(_values_per_block)); + } + if (_mini_blocks_per_block == 0) { + return Status::InvalidArgument("Cannot have zero miniblock per block"); + } + _values_per_mini_block = _values_per_block / _mini_blocks_per_block; + if (_values_per_mini_block == 0) { + return Status::InvalidArgument("Cannot have zero value per miniblock"); + } + if (_values_per_mini_block % 32 != 0) { + return Status::InvalidArgument( + "The number of values in a miniblock must be multiple of 32, but it's " + + std::to_string(_values_per_mini_block)); + } + // Encoded counts are external ULEB32 values. Bound them by the page's advertised logical + // values before any vector uses the count; optional levels may only reduce this upper bound. + if (UNLIKELY(_total_value_count > _expected_values)) { + return Status::Corruption("Parquet delta header advertises {} values, page allows {}", + _total_value_count, _expected_values); + } + _total_values_remaining = _total_value_count; + _delta_bit_widths.clear(); + // init as empty property + _block_initialized = false; + _delta_bit_width = 0; + _values_remaining_current_mini_block = 0; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_block() { + DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF"; + if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) { + return Status::IOError("Init block eof"); + } + + // One byte follows for each miniblock. Defer allocation until a block is actually consumed so + // a one-value page cannot turn an irrelevant malicious miniblock count into a large resize. + if (UNLIKELY(_mini_blocks_per_block > static_cast(_bit_reader->bytes_left()))) { + return Status::Corruption("Parquet delta miniblock count {} exceeds remaining {} bytes", + _mini_blocks_per_block, _bit_reader->bytes_left()); + } + _delta_bit_widths.resize(_mini_blocks_per_block); + + // read the bitwidth of each miniblock + uint8_t* bit_width_data = _delta_bit_widths.data(); + for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { + if (!_bit_reader->GetAligned(1, bit_width_data + i)) { + return Status::IOError("Decode bit-width EOF"); + } + // Note that non-conformant bitwidth entries are allowed by the Parquet spec + // for extraneous miniblocks in the last block (GH-14923), so we check + // the bitwidths when actually using them (see InitMiniBlock()). + } + _mini_block_idx = 0; + _block_initialized = true; + RETURN_IF_ERROR(_init_mini_block(bit_width_data[0])); + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_mini_block(int bit_width) { + if (bit_width > kMaxDeltaBitWidth) [[unlikely]] { + return Status::InvalidArgument("delta bit width larger than integer bit width"); + } + _delta_bit_width = bit_width; + _values_remaining_current_mini_block = _values_per_mini_block; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, + uint32_t* out_num_values) { + num_values = std::min(num_values, _total_values_remaining); + if (num_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + uint32_t i = 0; + while (i < num_values) { + if (_values_remaining_current_mini_block == 0) [[unlikely]] { + if (!_block_initialized) [[unlikely]] { + buffer[i++] = _last_value; + DCHECK_EQ(i, 1); // we're at the beginning of the page + if (i == num_values) { + // When block is uninitialized and i reaches num_values we have two + // different possibilities: + // 1. _total_value_count == 1, which means that the page may have only + // one value (encoded in the header), and we should not initialize + // any block. + // 2. _total_value_count != 1, which means we should initialize the + // incoming block for subsequent reads. + if (_total_value_count != 1) { + RETURN_IF_ERROR(_init_block()); + } + break; + } + RETURN_IF_ERROR(_init_block()); + } else { + ++_mini_block_idx; + if (_mini_block_idx < _mini_blocks_per_block) { + RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx])); + } else { + RETURN_IF_ERROR(_init_block()); + } + } + } + + uint32_t values_decode = std::min(_values_remaining_current_mini_block, num_values - i); + for (uint32_t j = 0; j < values_decode; ++j) { + if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) { + return Status::IOError("Get batch EOF"); + } + } + for (int j = 0; j < values_decode; ++j) { + // Addition between min_delta, packed int and last_value should be treated as + // unsigned addition. Overflow is as expected. + buffer[i + j] = static_cast(_min_delta) + static_cast(buffer[i + j]) + + static_cast(_last_value); + _last_value = buffer[i + j]; + } + _values_remaining_current_mini_block -= values_decode; + i += values_decode; + } + _total_values_remaining -= num_values; + + if (_total_values_remaining == 0) [[unlikely]] { + int64_t padding_bits = 0; + // Footer-controlled miniblock counts can exceed 32-bit products. Widen before multiplying; + // BitReader::Advance then verifies that the full declared padding exists in the stream. + RETURN_IF_ERROR(detail::checked_delta_padding_bits(cast_set(_delta_bit_width), + _values_remaining_current_mini_block, + &padding_bits)); + if (!_bit_reader->Advance(padding_bits)) { + return Status::IOError("Skip padding EOF"); + } + _values_remaining_current_mini_block = 0; + } + *out_num_values = num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp new file mode 100644 index 00000000000000..1f1166421a4035 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +// Dictionary decoders retain only encoded physical values and the index-stream cursor. Logical +// interpretation is deliberately delegated to DataTypeSerDe through decode_dictionary(). +template +class FixLengthDictDecoder final : public BaseDictDecoder { +public: + FixLengthDictDecoder() = default; + ~FixLengthDictDecoder() override = default; + + size_t dictionary_size() const override { return _num_dictionary_values; } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dict.get(), _num_dictionary_values, + static_cast(_type_length)); + } + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override { + if (UNLIKELY(_type_length <= 0 || length < 0 || + num_values > std::numeric_limits::max() / + static_cast(_type_length) || + num_values * static_cast(_type_length) != + static_cast(length))) { + return Status::Corruption("Wrong dictionary data for fixed length type"); + } + if (UNLIKELY(dict == nullptr)) { + return Status::Corruption("Fixed-length Parquet dictionary is null"); + } + _dict = std::move(dict); + _num_dictionary_values = num_values; + ++_dictionary_generation; + return Status::OK(); + } + +private: + size_t _num_dictionary_values = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp new file mode 100644 index 00000000000000..28d8154f9db10d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +#include + +#include "util/cpu_info.h" + +namespace doris::format::parquet::native { + +Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + // Truncated PLAIN pages retain the public error keyword used by corrupt-file regression + // checks, while the bounds check still prevents the native decoder from reading past data. + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, + num_values, static_cast(_type_length))); + _offset += byte_size; + return Status::OK(); +} + +Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet plain selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError("Unexpected end of stream in Parquet plain selection decoder"); + } + const auto* values = reinterpret_cast(_data->data) + _offset; + _offset += input_bytes; + const size_t selected_bytes = selection.selected_values * value_width; + constexpr size_t MIN_FRAGMENTED_RANGES = 8; + constexpr size_t MAX_AVERAGE_RANGE_VALUES = 4; + const bool fragmented = + selection.ranges.size() >= MIN_FRAGMENTED_RANGES && + selection.selected_values <= selection.ranges.size() * MAX_AVERAGE_RANGE_VALUES; + if (fragmented && selected_bytes <= static_cast(CpuInfo::get_l2_cache_size())) { + _selected_values.resize(selected_bytes); + size_t output_offset = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first + range.count <= selection.total_values); + const size_t range_bytes = range.count * value_width; + memcpy(_selected_values.data() + output_offset, values + range.first * value_width, + range_bytes); + output_offset += range_bytes; + } + DORIS_CHECK_EQ(output_offset, selected_bytes); + // Tiny alternating runs make every consumer re-enter conversion and grow its destination + // column per range. A cache-resident gather keeps the page access sequential and restores + // one atomic conversion batch without retaining page-sized scratch for dense selections. + return consumer.consume(_selected_values.data(), selection.selected_values, value_width); + } + _selected_values.clear(); + // PLAIN pages are random-access fixed-width spans. Keep those page bytes pinned while the + // consumer gathers directly into the final column, otherwise sparse scans pay for a second + // selected-width buffer and copy before materialization. + return consumer.consume_selected(values, value_width, selection.ranges); +} + +Status FixLengthPlainDecoder::skip_values(size_t num_values) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + _offset += num_values * value_width; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h new file mode 100644 index 00000000000000..14913217e2e0de --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +class FixLengthPlainDecoder final : public Decoder { +public: + FixLengthPlainDecoder() = default; + ~FixLengthPlainDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_selected_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _selected_values.capacity(); } + size_t active_scratch_bytes() const override { return _selected_values.size(); } + +private: + std::vector _selected_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp new file mode 100644 index 00000000000000..618d6ebf9f6d45 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_decoder.h" + +#include + +#include +#include + +#include "common/cast_set.h" +#include "util/bit_stream_utils.inline.h" +#include "util/bit_util.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { + +static constexpr size_t V1_LEVEL_SIZE = 4; + +Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels) { + _encoding = encoding; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + switch (encoding) { + case tparquet::Encoding::RLE: { + if (slice->size < V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + + uint8_t* data = (uint8_t*)slice->data; + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > slice->size - V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + _rle_decoder = RleBatchDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + + slice->data += V1_LEVEL_SIZE + num_bytes; + slice->size -= V1_LEVEL_SIZE + num_bytes; + break; + } + case tparquet::Encoding::BIT_PACKED: { + // Header counts are uint32_t and can overflow before the byte bound check. Widen first so + // a forged level count cannot wrap to a small slice and desynchronize following values. + const uint64_t num_bits = static_cast(num_levels) * _bit_width; + const size_t num_bytes = static_cast((num_bits + 7) / 8); + if (num_bytes > slice->size) { + return Status::Corruption("Wrong parquet level format"); + } + if (num_bytes > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet BIT_PACKED level stream is too large"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, cast_set(num_bytes)); + + slice->data += num_bytes; + slice->size -= num_bytes; + break; + } + default: + return Status::IOError("Unsupported encoding for parquet level"); + } + return Status::OK(); +} + +Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t num_levels) { + _encoding = tparquet::Encoding::RLE; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + size_t byte_length = levels.size; + _rle_decoder = RleBatchDecoder((uint8_t*)levels.data, cast_set(byte_length), + _bit_width); + return Status::OK(); +} + +size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + _can_rewind = false; + // toto template. + if (_encoding == tparquet::Encoding::RLE) { + n = std::min((size_t)_num_levels, n); + size_t num_decoded = 0; + if (_has_buffered_level && n > 0) { + if (!accept_level(_buffered_level)) { + return 0; + } + levels[num_decoded++] = _buffered_level; + _has_buffered_level = false; + } + if (num_decoded < n) { + const size_t remaining = n - num_decoded; + _rle_scratch.resize(remaining); + const size_t batch_decoded = + _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); + for (size_t i = 0; i < batch_decoded; ++i) { + const level_t level = cast_set(_rle_scratch[i]); + if (!accept_level(level)) { + return 0; + } + levels[num_decoded + i] = level; + } + num_decoded += batch_decoded; + } + _num_levels -= num_decoded; + _can_rewind = false; + return num_decoded; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + n = std::min((size_t)_num_levels, n); + size_t decoded = 0; + for (; decoded < n; ++decoded) { + level_t level = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &level)) { + break; + } + if (!accept_level(level)) { + return 0; + } + levels[decoded] = level; + } + _num_levels -= decoded; + return decoded; + } + return 0; +} + +size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { + DORIS_CHECK(val != nullptr); + _can_rewind = false; + max_run = std::min(max_run, _num_levels); + if (max_run == 0) { + return 0; + } + if (_encoding == tparquet::Encoding::RLE) { + size_t decoded = 0; + if (_has_buffered_level) { + *val = _buffered_level; + _has_buffered_level = false; + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } else { + uint16_t first = 0; + if (_rle_decoder.GetBatch(&first, 1) != 1) { + return 0; + } + *val = cast_set(first); + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } + while (decoded < max_run) { + const int32_t repeats = _rle_decoder.NextNumRepeats(); + if (repeats > 0) { + const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (!accept_level(repeated)) return 0; + if (repeated != *val) break; + const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); + _rle_decoder.GetRepeatedValue(consume); + decoded += consume; + continue; + } + if (_rle_decoder.NextNumLiterals() == 0) break; + uint16_t literal = 0; + if (!_rle_decoder.GetLiteralValues(1, &literal)) break; + const level_t next = cast_set(literal); + if (!accept_level(next)) return 0; + if (next != *val) { + // Batch RLE has no physical rewind; retain the one-value lookahead logically. + _buffered_level = next; + _has_buffered_level = true; + break; + } + ++decoded; + } + _num_levels -= decoded; + _can_rewind = false; + return decoded; + } + if (_encoding != tparquet::Encoding::BIT_PACKED || + !_bit_packed_decoder.GetValue(_bit_width, val)) { + return 0; + } + if (!accept_level(*val)) { + return 0; + } + size_t decoded = 1; + while (decoded < max_run) { + level_t next = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { + break; + } + if (!accept_level(next)) return 0; + if (next != *val) { + // The lookahead belongs to the following run, so cursor APIs must leave it unread. + _bit_packed_decoder.Rewind(_bit_width); + break; + } + ++decoded; + } + _num_levels -= decoded; + return decoded; +} + +level_t LevelDecoder::get_next() { + if (_num_levels == 0) { + return -1; + } + level_t next = -1; + bool decoded = false; + if (_encoding == tparquet::Encoding::RLE) { + if (_has_buffered_level) { + next = _buffered_level; + _has_buffered_level = false; + decoded = true; + } else { + uint16_t value = 0; + decoded = _rle_decoder.GetBatch(&value, 1) == 1; + next = cast_set(value); + } + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + decoded = _bit_packed_decoder.GetValue(_bit_width, &next); + } + if (!decoded) { + return -1; + } + if (!accept_level(next)) { + return -1; + } + --_num_levels; + _last_level = next; + _can_rewind = true; + return next; +} + +void LevelDecoder::rewind_one() { + if (_encoding == tparquet::Encoding::RLE) { + DORIS_CHECK(_can_rewind && !_has_buffered_level); + _buffered_level = _last_level; + _has_buffered_level = true; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + DORIS_CHECK(_can_rewind); + _bit_packed_decoder.Rewind(_bit_width); + } else { + return; + } + // Rewinding restores one advertised level as well as its encoded bits. + ++_num_levels; + _can_rewind = false; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h new file mode 100644 index 00000000000000..9dcff491eb0759 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/common.h" +#include "util/bit_stream_utils.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class LevelDecoder { +public: + LevelDecoder() = default; + ~LevelDecoder() = default; + + Status init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels); + + Status init_v2(const Slice& levels, level_t max_level, uint32_t num_levels); + + inline bool has_levels() const { return _num_levels > 0; } + + size_t get_levels(level_t* levels, size_t n); + + size_t get_next_run(level_t* val, size_t max_run); + + level_t get_next(); + + void rewind_one(); + + void release_scratch(size_t max_retained_bytes) { + if (_rle_scratch.capacity() * sizeof(uint16_t) > max_retained_bytes) { + std::vector().swap(_rle_scratch); + } + } + size_t retained_scratch_bytes() const { return _rle_scratch.capacity() * sizeof(uint16_t); } + size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } + +private: + bool accept_level(level_t level) { + if (level >= 0 && level <= _max_level) { + return true; + } + // Bit width rounds up to a power of two, so encoded values can fit the bit stream while + // still exceeding the schema maximum. Poison the decoder before any caller can use them. + _num_levels = 0; + _has_buffered_level = false; + _can_rewind = false; + return false; + } + + tparquet::Encoding::type _encoding; + level_t _bit_width = 0; + level_t _max_level = 0; + uint32_t _num_levels = 0; + RleBatchDecoder _rle_decoder; + std::vector _rle_scratch; + BitReader _bit_packed_decoder; + bool _has_buffered_level = false; + bool _can_rewind = false; + level_t _buffered_level = -1; + level_t _last_level = -1; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp new file mode 100644 index 00000000000000..06b71582d7a2be --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_reader.h" + +#include +#include + +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" + +namespace doris::format::parquet::native { + +class LevelReader::Impl { +public: + virtual ~Impl() = default; + + virtual Status init() = 0; + virtual Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) = 0; + virtual ColumnChunkReaderStatistics statistics() = 0; +}; + +template +class LevelReaderImpl final : public LevelReader::Impl { +public: + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, std::string page_cache_file_key, + ParquetReaderCompat compat) + : _file(std::move(file)), + _column_chunk(std::move(column_chunk)), + _field(field), + _total_rows(total_rows), + _max_buffer_size(max_buffer_size), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)), + _compat(compat) {} + + Status init() override { + DORIS_CHECK(_file != nullptr); + DORIS_CHECK(_field != nullptr); + const auto& metadata = _column_chunk.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(metadata, _file->size(), + _compat.parquet_816_padding, &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_size = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); + auto* tracing_reader = typeid_cast(_file.get()); + if ((tracing_reader != nullptr && + typeid_cast(tracing_reader->inner_reader().get()) != + nullptr) || + typeid_cast(_file.get()) != nullptr) { + prefetch_buffer_size = 0; + } + _stream = std::make_unique(_file, chunk_start, chunk_size, + prefetch_buffer_size); + _chunk_reader = std::make_unique>( + _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key, + _compat.data_page_v2_always_compressed), + &chunk_range); + return _chunk_reader->init(); + } + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) override { + DORIS_CHECK(repetition_levels != nullptr); + DORIS_CHECK(definition_levels != nullptr); + DORIS_CHECK(rows_read != nullptr); + repetition_levels->clear(); + definition_levels->clear(); + *rows_read = 0; + if (_current_row > _total_rows || rows > _total_rows - _current_row) { + return Status::Corruption("Parquet level reader requested rows [{}, {}) of {}", + _current_row, _current_row + rows, _total_rows); + } + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR( + read_nested_rows(rows, repetition_levels, definition_levels, rows_read)); + } else { + RETURN_IF_ERROR(read_flat_rows(rows, repetition_levels, definition_levels, rows_read)); + } + _current_row += *rows_read; + return Status::OK(); + } + + ColumnChunkReaderStatistics statistics() override { return _chunk_reader->statistics(); } + +private: + Status read_flat_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + const size_t read_now = std::min( + rows - *rows_read, static_cast(_chunk_reader->remaining_num_values())); + if (read_now == 0) { + // Zero-value data pages are legal and do not contribute to the chunk cardinality. + // Advance them before applying the no-progress guard used for truncated chunks. + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + continue; + } + return Status::Corruption("Parquet flat level reader made no progress"); + } + RETURN_IF_ERROR( + _chunk_reader->read_levels(read_now, repetition_levels, definition_levels)); + *rows_read += read_now; + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + return Status::OK(); + } + + Status read_nested_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(_current_row + *rows_read)); + const size_t start_level = definition_levels->size(); + size_t loaded_rows = 0; + bool crosses_page = false; + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows( + *repetition_levels, rows - *rows_read, &loaded_rows, &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(*definition_levels, start_level)); + while (crosses_page) { + const size_t continuation_start = definition_levels->size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(*repetition_levels, + &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR( + _chunk_reader->skip_nested_values(*definition_levels, continuation_start)); + } + if (loaded_rows == 0) { + return Status::Corruption("Parquet nested level reader made no progress"); + } + *rows_read += loaded_rows; + } + return Status::OK(); + } + + io::FileReaderSPtr _file; + tparquet::ColumnChunk _column_chunk; + NativeFieldSchema* _field = nullptr; + size_t _total_rows = 0; + size_t _max_buffer_size = 0; + io::IOContext* _io_ctx = nullptr; + bool _enable_page_cache = false; + std::string _page_cache_file_key; + ParquetReaderCompat _compat; + size_t _current_row = 0; + std::unique_ptr _stream; + std::unique_ptr> _chunk_reader; +}; + +Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, + std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(field != nullptr); + std::unique_ptr impl; + if (field->repetition_level > 0) { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } else { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } + RETURN_IF_ERROR(impl->init()); + reader->reset(new LevelReader(std::move(impl))); + return Status::OK(); +} + +LevelReader::LevelReader(std::unique_ptr impl) : _impl(std::move(impl)) {} + +LevelReader::~LevelReader() = default; + +Status LevelReader::read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + return _impl->read_rows(rows, repetition_levels, definition_levels, rows_read); +} + +Status LevelReader::skip_rows(size_t rows) { + size_t rows_read = 0; + RETURN_IF_ERROR( + read_rows(rows, &_skip_repetition_levels, &_skip_definition_levels, &rows_read)); + if (rows_read != rows) { + return Status::Corruption("Parquet level reader skipped {} of {} rows", rows_read, rows); + } + return Status::OK(); +} + +ColumnChunkReaderStatistics LevelReader::statistics() { + return _impl->statistics(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h new file mode 100644 index 00000000000000..11117d25dcb252 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { + +// A physical-leaf reader that advances all three Parquet streams (repetition levels, definition +// levels and encoded values) but retains only the two level streams. It is used for shape-only +// operations such as COUNT(nullable_col), where materializing a large BYTE_ARRAY value would be +// both unnecessary and potentially unbounded. +// +// This deliberately shares ColumnChunkReader with the value path. Page V1/V2 parsing, +// decompression, dictionary-page handling, page cache semantics and corruption checks therefore +// cannot drift between an aggregate shortcut and an ordinary scan. +class LevelReader { +public: + class Impl; + + static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + std::unique_ptr* reader); + + ~LevelReader(); + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read); + Status skip_rows(size_t rows); + ColumnChunkReaderStatistics statistics(); + +private: + explicit LevelReader(std::unique_ptr impl); + + std::unique_ptr _impl; + // COUNT range gaps can call skip repeatedly as the adaptive row cap changes. Keep these + // throw-away level buffers on the persistent reader so only their logical sizes are reset. + std::vector _skip_repetition_levels; + std::vector _skip_definition_levels; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp new file mode 100644 index 00000000000000..37f682cbd4d9d5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/page_reader.h" + +#include +#include +#include +#include + +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" +#include "util/thrift_util.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; + +template +Status PageReader::_validate_page_header(uint32_t header_size) const { + if (UNLIKELY(_cur_page_header.compressed_page_size < 0 || + _cur_page_header.uncompressed_page_size < 0)) { + return Status::Corruption("Parquet page has a negative compressed or uncompressed size"); + } + if (UNLIKELY(header_size > _end_offset - _offset || + static_cast(_cur_page_header.compressed_page_size) > + _end_offset - _offset - header_size)) { + // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. + return Status::Corruption("Parquet page payload exceeds its column chunk"); + } + + const bool has_v1 = _cur_page_header.__isset.data_page_header; + const bool has_v2 = _cur_page_header.__isset.data_page_header_v2; + const bool has_dictionary = _cur_page_header.__isset.dictionary_page_header; + const bool has_index = _cur_page_header.__isset.index_page_header; + bool matching_layout = false; + switch (_cur_page_header.type) { + case tparquet::PageType::DATA_PAGE: + matching_layout = has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DATA_PAGE_V2: + matching_layout = has_v2 && !has_v1 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DICTIONARY_PAGE: + matching_layout = has_dictionary && !has_v1 && !has_v2 && !has_index; + break; + case tparquet::PageType::INDEX_PAGE: + matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; + break; + default: + // Forward-compatible auxiliary pages have no known page-specific member. Their bounded + // payload can be skipped safely; accepting a known member here would let the enum and + // decoder layout disagree. + matching_layout = !has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + } + if (UNLIKELY(!matching_layout)) { + // The type discriminant owns the only valid union member. Guessing from an optional header + // lets a malformed page use one layout for validation and another for decoding. + return Status::Corruption("Parquet page type does not match its page-header layout"); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + const auto& v2 = _cur_page_header.data_page_header_v2; + if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || + v2.repetition_levels_byte_length < 0 || + v2.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has negative counts or level sizes"); + } + if (UNLIKELY(v2.num_nulls > v2.num_values || v2.num_rows > v2.num_values)) { + return Status::Corruption( + "Parquet data page v2 null or row count exceeds its value count"); + } + const uint64_t level_bytes = static_cast(v2.repetition_levels_byte_length) + + static_cast(v2.definition_levels_byte_length); + if (UNLIKELY(level_bytes > static_cast(_cur_page_header.compressed_page_size) || + level_bytes > + static_cast(_cur_page_header.uncompressed_page_size))) { + return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); + } + } else if (_cur_page_header.type == tparquet::PageType::DATA_PAGE && + UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { + return Status::Corruption("Parquet data page has a negative value count"); + } else if (_cur_page_header.type == tparquet::PageType::DICTIONARY_PAGE && + UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { + return Status::Corruption("Parquet dictionary page has a negative value count"); + } + return Status::OK(); +} + +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _metadata(metadata), + _page_read_ctx(page_read_ctx), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + _page_cache_key_builder.init(_page_read_ctx.page_cache_file_key); + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index != nullptr && _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; + } +} + +template +void PageReader::_reconcile_offset_index_location( + uint64_t header_offset, uint32_t header_size) { + if constexpr (!OFFSET_INDEX) { + return; + } + if (_offset_index == nullptr || _page_index >= _offset_index->page_locations.size()) { + return; + } + const auto& location = _offset_index->page_locations[_page_index]; + const bool data_page = _cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2; + const uint64_t serialized_size = static_cast(header_size) + + static_cast(_cur_page_header.compressed_page_size); + const bool matches = data_page && location.offset >= 0 && location.compressed_page_size > 0 && + header_offset == static_cast(location.offset) && + serialized_size == static_cast(location.compressed_page_size); + if (!matches && (data_page || (location.offset >= 0 && + header_offset >= static_cast(location.offset)))) { + // OffsetIndex is optional. Once its data-page rectangle disagrees with the serialized + // page, continue sequentially so a shifted location cannot redirect the next seek. + _offset_index = nullptr; + } +} + +template +void PageReader::_update_sequential_row_range() { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return; + } + } + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } +} + +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } + if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { + return Status::IOError("Out-of-bounds Access"); + } + if (UNLIKELY(_offset != _next_header_offset)) { + return Status::IOError("Wrong header position, should seek to a page header first"); + } + if (UNLIKELY(_state != INITIALIZED)) { + return Status::IOError("Should skip or load current page to get next page"); + } + + _page_statistics.page_read_counter += 1; + + // Parse page header from file; header bytes are saved for possible cache insertion + const uint8_t* page_header_buf = nullptr; + size_t max_size = _end_offset - _offset; + size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size); + const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20; + uint32_t real_header_size = 0; + + // Try a header-only lookup in the page cache. Cached pages store + // header + optional v2 levels + uncompressed payload, so we can + // parse the page header directly from the cached bytes and avoid + // a file read for the header. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + PageCacheHandle handle; + StoragePageCache::CacheKey key = make_page_cache_key(static_cast(_offset)); + if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) { + // Parse header directly from cached data + _page_cache_handle = std::move(handle); + Slice s = _page_cache_handle.data(); + real_header_size = cast_set(s.size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Thrift does not clear absent optional fields when reusing an output object. + _cur_page_header = tparquet::PageHeader {}; + auto st = deserialize_thrift_msg(reinterpret_cast(s.data), + &real_header_size, true, &_cur_page_header); + if (!st.ok()) return st; + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + // Increment page cache counters for a true cache hit on header+payload + _page_statistics.page_cache_hit_counter += 1; + // Detect whether the cached payload is compressed or decompressed and record + bool is_cache_payload_decompressed = should_cache_decompressed( + &_cur_page_header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (is_cache_payload_decompressed) { + _page_statistics.page_cache_decompressed_hit_counter += 1; + } else { + _page_statistics.page_cache_compressed_hit_counter += 1; + } + + _is_cache_payload_decompressed = is_cache_payload_decompressed; + + // Save header bytes for later use (e.g., to insert updated cache entries) + _header_buf.assign(s.data, s.data + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); + } else { + _page_statistics.page_cache_missing_counter += 1; + // Clear any existing cache handle on miss to avoid holding stale handle + _page_cache_handle = PageCacheHandle(); + } + } + // NOTE: page cache lookup for *decompressed* page data is handled in + // ColumnChunkReader::load_page_data(). PageReader should only be + // responsible for parsing the header bytes from the file and saving + // them in `_header_buf` for possible later insertion into the cache. + while (true) { + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + header_size = std::min(header_size, max_size); + { + SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time); + RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx)); + } + real_header_size = cast_set(header_size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Reset on every retry as a partial deserialize can otherwise leak a stale union member. + _cur_page_header = tparquet::PageHeader {}; + auto st = + deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); + if (st.ok()) { + break; + } + if (_offset + header_size >= _end_offset || real_header_size > MAX_PAGE_HEADER_SIZE) { + return Status::IOError( + "Failed to deserialize parquet page header. offset: {}, " + "header size: {}, end offset: {}, real header size: {}", + _offset, header_size, _end_offset, real_header_size); + } + header_size <<= 2; + } + + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + + // Save header bytes for possible cache insertion later + _header_buf.assign(page_header_buf, page_header_buf + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status PageReader::get_page_data(Slice& slice) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::IOError("Should generate page header first to load current page data"); + } + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + slice.size = _cur_page_header.compressed_page_size; + RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); + _offset += slice.size; + _state = DATA_LOADED; + return Status::OK(); +} + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h new file mode 100644 index 00000000000000..da2c066891bda7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "storage/cache/page_cache.h" +#include "util/block_compression.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +/** + * Use to deserialize parquet page header, and get the page data in iterator interface. + */ + +// Session-level options for parquet page reading/caching. +struct ParquetPageReadContext { + // A default-constructed context has no stable file identity, so cache lookup must stay off. + bool enable_parquet_file_page_cache = false; + std::string page_cache_file_key; + bool data_page_v2_always_compressed = false; + ParquetPageReadContext() = default; + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key, + bool data_page_v2_always_compressed = false) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache && + !page_cache_file_key.empty()), + page_cache_file_key(std::move(page_cache_file_key)), + data_page_v2_always_compressed(data_page_v2_always_compressed) {} +}; + +inline bool should_cache_decompressed(const tparquet::PageHeader* header, + const tparquet::ColumnMetaData& metadata, + bool data_page_v2_always_compressed = false) { + // Data Page V2 declares its payload representation independently of the column codec. A warm + // hit must never send an explicitly uncompressed cached payload through the codec again. + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed && + !data_page_v2_always_compressed) { + return true; + } + if (header->compressed_page_size <= 0) return true; + if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; + if (header->uncompressed_page_size == 0) return true; + + double ratio = static_cast(header->uncompressed_page_size) / + static_cast(header->compressed_page_size); + return ratio <= config::parquet_page_cache_decompress_threshold; +} + +class ParquetPageCacheKeyBuilder { +public: + void init(std::string file_key) { _file_key_prefix = std::move(file_key); } + StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { + return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); + } + +private: + std::string _file_key_prefix; +}; + +template +class PageReader { +public: + struct PageStatistics { + int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + }; + + PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; + + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return _page_index + 1 < _offset_index->page_locations.size(); + } + } + return _offset < _end_offset; + } + + Status parse_page_header(); + + Status next_page() { + _page_statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + if (UNLIKELY(_page_index + 1 >= _offset_index->page_locations.size())) { + return Status::Corruption("Parquet OffsetIndex has no next page location"); + } + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + return Status::OK(); + } + } + + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + _offset = _next_header_offset; + _state = INITIALIZED; + + return Status::OK(); + } + + Status skip_auxiliary_page() { + if constexpr (OFFSET_INDEX) { + // OffsetIndex enumerates data pages only, so an auxiliary physical page must not + // consume a logical page-location entry while advancing to its payload end. + skip_page_data(); + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status get_page_header(const tparquet::PageHeader** page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + *page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_page_data(Slice& slice); + + // Skip page data and update offset (used when data is loaded from cache) + void skip_page_data() { + if (_state == HEADER_PARSED) { + _offset += _cur_page_header.compressed_page_size; + _state = DATA_LOADED; + } + } + + const std::vector& header_bytes() const { return _header_buf; } + // header start offset for current page + int64_t header_start_offset() const { + return static_cast(_next_header_offset) - static_cast(_last_header_size) - + static_cast(_cur_page_header.compressed_page_size); + } + uint64_t file_end_offset() const { return _end_offset; } + bool cached_decompressed() const { + return should_cache_decompressed(&_cur_page_header, _metadata, + _page_read_ctx.data_page_v2_always_compressed); + } + + PageStatistics& page_statistics() { return _page_statistics; } + + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } + + // Returns whether the current page's cache payload is decompressed + bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; } + + size_t start_row() const { return _start_row; } + + size_t end_row() const { return _end_row; } + + bool has_active_offset_index() const { + if constexpr (OFFSET_INDEX) { + return _offset_index != nullptr; + } + return false; + } + + void discard_offset_index() { + if constexpr (OFFSET_INDEX) { + _offset_index = nullptr; + } + } + + // Accessors for cache handle + bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; } + const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; } + StoragePageCache::CacheKey make_page_cache_key(int64_t offset) const { + return _page_cache_key_builder.make_key(_end_offset, offset); + } + +private: + Status _validate_page_header(uint32_t header_size) const; + void _reconcile_offset_index_location(uint64_t header_offset, uint32_t header_size); + void _update_sequential_row_range(); + + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + PageStatistics _page_statistics; + + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. + uint64_t _offset = 0; + // this page offset in file location. + uint64_t _start_offset = 0; + uint64_t _end_offset = 0; + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // Column metadata for this column chunk + const tparquet::ColumnMetaData& _metadata; + // Session-level parquet page cache options + ParquetPageReadContext _page_read_ctx; + // for page index + size_t _page_index = 0; + const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; + bool _is_cache_payload_decompressed = true; + + // Page cache members + ParquetPageCacheKeyBuilder _page_cache_key_builder; + doris::PageCacheHandle _page_cache_handle; + // stored header bytes when cache miss so we can insert header+payload into cache + std::vector _header_buf; + // last parsed header size in bytes + uint32_t _last_header_size = 0; +}; + +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& ctx, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>( + reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp new file mode 100644 index 00000000000000..8aa79ecb7e7e23 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -0,0 +1,767 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { +namespace { + +constexpr size_t MAX_RETAINED_BATCH_SCRATCH_BYTES = 4UL << 20; + +DataTypePtr projected_type(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection) { + if (!format::is_partial_projection(projection)) { + return schema.type; + } + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + return schema.type; + case ParquetColumnSchemaKind::STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projection->children.size()); + child_names.reserve(projection->children.size()); + for (const auto& child_projection : projection->children) { + const auto child_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != schema.children.end()); + child_types.push_back(make_nullable(projected_type(**child_it, &child_projection))); + child_names.push_back((*child_it)->name); + } + DataTypePtr type = std::make_shared(child_types, child_names); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto* child_projection = + format::find_child_projection(projection, schema.children[0]->local_id); + DORIS_CHECK(child_projection != nullptr); + DataTypePtr type = std::make_shared( + projected_type(*schema.children[0], child_projection)); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::MAP: { + DORIS_CHECK(schema.children.size() == 2); + const auto* value_projection = + format::find_child_projection(projection, schema.children[1]->local_id); + DORIS_CHECK(value_projection != nullptr); + DataTypePtr type = std::make_shared( + make_nullable(schema.children[0]->type), + make_nullable(projected_type(*schema.children[1], value_projection))); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + } + DORIS_CHECK(false); + return nullptr; +} + +const NativeFieldSchema* find_child_field(const NativeFieldSchema& parent, + const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const NativeFieldSchema& field) { + return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || + field.name == child.name; + }); + return field_it == parent.children.end() ? nullptr : &*field_it; +} + +void collect_physical_subtree_ids(const NativeFieldSchema& field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + ids->insert(field.get_column_id()); + for (const auto& child : field.children) { + collect_physical_subtree_ids(child, ids); + } +} + +void collect_projected_ids(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const NativeFieldSchema& native_field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + if (!format::is_partial_projection(projection)) { + return; + } + for (const auto& child_projection : projection->children) { + const auto schema_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(schema_it != schema.children.end()); + const NativeFieldSchema* child_field = find_child_field(native_field, **schema_it); + DORIS_CHECK(child_field != nullptr); + if (format::is_full_projection(&child_projection)) { + // A full child path is a request for its complete physical subtree. Keeping only the + // child group id makes its grandchildren SkipReadingReaders and silently defaults + // STRUCT fields (or breaks ARRAY/MAP shape invariants). + collect_physical_subtree_ids(*child_field, ids); + } else { + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } + } + if (schema.kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(!native_field.children.empty()); + // MAP entry existence and offsets are owned by the key stream even for value-only + // projections. Keep the key reader live so the native complex reader can validate + // key/value entry alignment while constructing offsets. + ids->insert(native_field.children[0].get_column_id()); + } +} + +Status append_non_null_dictionary_values(MutableColumnPtr& target, MutableColumnPtr values) { + DORIS_CHECK(target); + DORIS_CHECK(values); + const size_t value_count = values->size(); + if (auto* nullable = check_and_get_column(*target); nullable != nullptr) { + nullable->get_nested_column().insert_range_from(*values, 0, value_count); + auto& null_map = nullable->get_null_map_data(); + null_map.resize_fill(null_map.size() + value_count, 0); + return Status::OK(); + } + target->insert_range_from(*values, 0, value_count); + return Status::OK(); +} + +} // namespace + +NativeColumnReader::NativeColumnReader(const ParquetColumnSchema& schema, + DataTypePtr projected_type, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(schema, std::move(projected_type), profile), + _nested(schema.kind != ParquetColumnSchemaKind::PRIMITIVE) {} + +NativeColumnReader::~NativeColumnReader() { + (void)sync_native_profile(); +} + +Status NativeColumnReader::create( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + if (reader == nullptr) { + return Status::InvalidArgument("Native parquet reader result is null"); + } + if (file == nullptr || metadata == nullptr) { + return Status::InvalidArgument("Native parquet file context is not initialized"); + } + if (row_group_id < 0 || + row_group_id >= static_cast(metadata->to_thrift().row_groups.size())) { + return Status::InvalidArgument("Invalid native parquet row group {}", row_group_id); + } + const auto& native_schema = metadata->schema(); + if (column_schema.local_id < 0 || column_schema.local_id >= native_schema.size()) { + return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", + column_schema.local_id, column_schema.name); + } + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + DORIS_CHECK(field != nullptr); + if (field->name != column_schema.name && + !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { + return Status::Corruption( + "Native/metadata parquet schema mismatch at column {}: native={}, arrow={}", + column_schema.local_id, field->name, column_schema.name); + } + + auto type = projected_type(column_schema, projection); + std::shared_ptr schema_node; + RETURN_IF_ERROR(build_native_schema_node(type, column_schema, &schema_node)); + std::set projected_ids; + collect_projected_ids(column_schema, projection, *field, &projected_ids); + + auto native_reader = std::unique_ptr( + new NativeColumnReader(column_schema, std::move(type), profile)); + RETURN_IF_ERROR(native_reader->init( + std::move(file), metadata, row_group_id, field, std::move(schema_node), + std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, + runtime_state, enable_page_cache, page_cache_file_key, enable_dictionary_filter)); + *reader = std::move(native_reader); + return Status::OK(); +} + +Status NativeColumnReader::init( + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(field != nullptr); + DORIS_CHECK(schema_node != nullptr); + const auto& row_group = metadata->to_thrift().row_groups[row_group_id]; + DORIS_CHECK(row_group.num_rows > 0); + _row_group_rows = row_group.num_rows; + _selected_ranges = selected_ranges; + DORIS_CHECK(!_selected_ranges.empty()); + for (const auto& range : _selected_ranges) { + DORIS_CHECK(range.start >= 0); + DORIS_CHECK(range.length > 0); + DORIS_CHECK(range.start + range.length <= _row_group_rows); + _row_ranges.add(segment_v2::RowRange(range.start, range.start + range.length)); + } + // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them + // avoids retaining the full N-column page-location map once per projected reader. + _offset_indexes = &offset_indexes; + _schema_node = std::move(schema_node); + _projected_column_ids = std::move(projected_column_ids); + _dictionary_filter_enabled = enable_dictionary_filter; + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + const size_t max_buffer_size = std::min(max_group_buffer, max_column_buffer); + RuntimeState* native_runtime_state = runtime_state; + const bool runtime_page_cache_enabled = + runtime_state == nullptr || + runtime_state->query_options().enable_parquet_file_page_cache; + if (runtime_page_cache_enabled != enable_page_cache) { + TQueryOptions query_options = + runtime_state == nullptr ? TQueryOptions() : runtime_state->query_options(); + query_options.__set_enable_parquet_file_page_cache(enable_page_cache); + _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); + native_runtime_state = _page_cache_runtime_state.get(); + } + const auto& thrift_metadata = metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::ColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids, page_cache_file_key, compat, + runtime_state != nullptr && runtime_state->enable_strict_mode())); + DORIS_CHECK(_native_reader != nullptr); + _skip_column = _type->create_column(); + return Status::OK(); +} + +Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, MutableColumnPtr& column, + const DataTypePtr& output_type, bool dictionary_ids, + int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(column); + DORIS_CHECK(output_type != nullptr); + DORIS_CHECK(rows_read != nullptr); + *rows_read = 0; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + ColumnPtr native_column(std::move(column)); + bool eof = false; + int64_t native_calls = 0; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + ++native_calls; + size_t loop_rows = 0; + RETURN_IF_ERROR(_native_reader->read_column_data( + native_column, output_type, _schema_node, filter, + static_cast(rows - *rows_read), &loop_rows, &eof, dictionary_ids)); + if (loop_rows == 0 && !eof) { + // A selected RowRanges plan may reject the current data page completely. V1 advances + // the page cursor and deliberately returns zero rows so the caller can request the + // next page. Bound consecutive empty transitions by the Row Group row count to retain + // a deterministic corruption exit if a decoder ever stops advancing. + if (++consecutive_empty_calls > _row_group_rows + 1) { + column = IColumn::mutate(std::move(native_column)); + return Status::Corruption("Native parquet reader made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + column = IColumn::mutate(std::move(native_column)); + if (_profile.native_read_calls != nullptr) { + COUNTER_UPDATE(_profile.native_read_calls, native_calls); + } + if (_nested && _profile.nested_batches != nullptr) { + COUNTER_UPDATE(_profile.nested_batches, 1); + } + // Retained-capacity inspection walks the native reader tree. Check it periodically instead of + // on every small batch; row-group destruction is still the hard lifetime bound for scratch. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } + if (*rows_read != rows) { + return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + return Status::OK(); +} + +Status NativeColumnReader::read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *rows_read = 0; + *used_filter = false; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + bool eof = false; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + size_t loop_rows = 0; + IColumn::Filter loop_filter; + bool loop_used = false; + RETURN_IF_ERROR(_native_reader->read_fixed_width_filter( + conjuncts, column_id, filter, static_cast(rows - *rows_read), + projected_column, &loop_filter, &loop_rows, &eof, &loop_used)); + if (!loop_used) { + if (UNLIKELY(*rows_read != 0)) { + // Footer encoding lists are untrusted. Once a prior page advanced the cursor, a + // typed fallback would restart the request at the wrong row, so reject the file + // instead of terminating the BE or returning shifted results. + return Status::Corruption( + "Parquet fixed-width predicate encoding changed after {} rows for column " + "{}", + *rows_read, _name); + } + row_filter->clear(); + return Status::OK(); + } + row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end()); + if (loop_rows == 0 && !eof) { + if (++consecutive_empty_calls > _row_group_rows + 1) { + return Status::Corruption( + "Native parquet fixed-width predicate made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + if (*rows_read != rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + *used_filter = true; + return Status::OK(); +} + +Status NativeColumnReader::validate_selected_span(int64_t rows) { + DORIS_CHECK(rows >= 0); + while (_selected_range_idx < _selected_ranges.size()) { + const auto& range = _selected_ranges[_selected_range_idx]; + const int64_t range_end = range.start + range.length; + if (_logical_row_position < range_end) { + break; + } + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + return Status::Corruption("Native parquet read past selected ranges for column {}", _name); + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start || + rows > range.start + range.length - _logical_row_position) { + return Status::Corruption( + "Native parquet read [{}, {}) crosses selected range [{}, {}) for column {}", + _logical_row_position, _logical_row_position + rows, range.start, + range.start + range.length, _name); + } + return Status::OK(); +} + +void NativeColumnReader::advance_selected_span(int64_t rows) { + _logical_row_position += rows; + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } +} + +Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(validate_selected_span(rows)); + RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); + advance_selected_span(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status NativeColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + DORIS_CHECK(_logical_row_position <= _row_group_rows - rows); + int64_t remaining = rows; + int64_t native_skipped_rows = 0; + while (remaining > 0) { + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + _logical_row_position += remaining; + break; + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start) { + const int64_t gap = std::min(remaining, range.start - _logical_row_position); + _logical_row_position += gap; + remaining -= gap; + continue; + } + const int64_t selected_rows = detail::bounded_native_lazy_skip_rows( + std::min(remaining, range.start + range.length - _logical_row_position)); + _skip_column->clear(); + // Pending skips can span many filtered batches and are replayed for every lazy column. + // Chunking here bounds each dense bitmap while preserving one logical scheduler skip. + _filter_scratch.assign(static_cast(selected_rows), 0); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, + _type, false, &rows_read)); + DORIS_CHECK(_skip_column->empty()); + DORIS_CHECK(rows_read == selected_rows); + _logical_row_position += rows_read; + native_skipped_rows += rows_read; + remaining -= rows_read; + } + update_reader_skip_rows(native_skipped_rows); + return Status::OK(); +} + +Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const size_t old_size = column->size(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, column, _type, + false, &rows_read)); + advance_selected_span(rows_read); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Native parquet selection appended {} rows, expected {} for column {}", + column->size() - old_size, selected_rows, _name); + } + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + *used_filter = false; + row_filter->clear(); + if (!_dictionary_filter_enabled) { + return Status::OK(); + } + *used_filter = true; + + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const bool nullable = _type->is_nullable(); + DataTypePtr id_type = std::make_shared(); + if (nullable) { + id_type = make_nullable(id_type); + } + if (!_dictionary_id_column) { + _dictionary_id_column = id_type->create_column(); + } + _dictionary_id_column->clear(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, + _dictionary_id_column, id_type, true, &rows_read)); + advance_selected_span(rows_read); + if (_dictionary_id_column->size() != selected_rows) { + return Status::Corruption( + "Native parquet dictionary reader appended {} rows, expected {} for {}", + _dictionary_id_column->size(), selected_rows, _name); + } + + const ColumnInt32* ids = nullptr; + const NullMap* null_map = nullptr; + if (const auto* nullable_ids = check_and_get_column(*_dictionary_id_column); + nullable_ids != nullptr) { + ids = check_and_get_column(nullable_ids->get_nested_column()); + null_map = &nullable_ids->get_null_map_data(); + } else { + ids = check_and_get_column(*_dictionary_id_column); + } + DORIS_CHECK(ids != nullptr); + + if (!_matched_dictionary_ids) { + _matched_dictionary_ids = ColumnInt32::create(); + } + _matched_dictionary_ids->clear(); + auto& matched_ids = assert_cast(*_matched_dictionary_ids).get_data(); + row_filter->reserve(selected_rows); + const auto& id_data = ids->get_data(); + for (size_t row = 0; row < selected_rows; ++row) { + bool keep = false; + if (null_map == nullptr || (*null_map)[row] == 0) { + const int32_t dictionary_id = id_data[row]; + if (dictionary_id < 0 || + static_cast(dictionary_id) >= dictionary_filter.size()) { + return Status::Corruption( + "Invalid parquet dictionary id {} for column {} with {} entries", + dictionary_id, _name, dictionary_filter.size()); + } + keep = dictionary_filter[static_cast(dictionary_id)] != 0; + if (keep) { + matched_ids.push_back(dictionary_id); + } + } + row_filter->push_back(keep ? 1 : 0); + } + + const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); + DORIS_CHECK(matched_id_column != nullptr); + auto matched_values = + DORIS_TRY(_native_reader->materialize_dictionary_values(matched_id_column, _type)); + RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(matched_values))); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(cast_set(matched_ids.size())); + update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + return Status::OK(); +} + +Status NativeColumnReader::select_with_fixed_width_filter( + const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_fixed_width_filter(batch_rows, filter_data, selected_rows == 0, + conjuncts, column_id, projected_column, row_filter, + &rows_read, used_filter)); + if (!*used_filter) { + return Status::OK(); + } + DORIS_CHECK_EQ(rows_read, batch_rows); + if (row_filter->size() != selected_rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} selected rows, expected {} for " + "{}", + row_filter->size(), selected_rows, _name); + } + advance_selected_span(rows_read); + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +void NativeColumnReader::flush_profile() { + record_page_fragments(sync_native_profile()); +} + +bool NativeColumnReader::crossed_page_since_last_batch() { + if (_native_reader == nullptr) { + return false; + } + const auto stats = _native_reader->column_statistics(); + bool crossed_page = false; + if (stats.leaf_page_read_counters.size() == _batch_leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + crossed_page |= + stats.leaf_page_read_counters[leaf] - _batch_leaf_page_read_counters[leaf] > 1; + } + } else { + // The first snapshot covers the first batch; later snapshots must keep the stable tree shape. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + crossed_page |= page_reads > 1; + } + } + _batch_leaf_page_read_counters = stats.leaf_page_read_counters; + return crossed_page; +} + +Result NativeColumnReader::dictionary_values() { + DORIS_CHECK(_native_reader != nullptr); + return _native_reader->dictionary_values(_type); +} + +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { + if (_profile.native_page_fragments != nullptr) { + COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); + } +} + +int64_t NativeColumnReader::sync_native_profile() { + if (_native_reader == nullptr) { + return 0; + } + const auto stats = _native_reader->column_statistics(); + const auto& reported = _reported_native_stats; + if (_profile.decompress_time != nullptr) { + COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); + } + if (_profile.decompress_count != nullptr) { + COUNTER_UPDATE(_profile.decompress_count, stats.decompress_cnt - reported.decompress_cnt); + } + if (_profile.decode_header_time != nullptr) { + COUNTER_UPDATE(_profile.decode_header_time, + stats.decode_header_time - reported.decode_header_time); + } + if (_profile.decode_value_time != nullptr) { + COUNTER_UPDATE(_profile.decode_value_time, + stats.decode_value_time - reported.decode_value_time); + } + if (_profile.decode_dictionary_time != nullptr) { + COUNTER_UPDATE(_profile.decode_dictionary_time, + stats.decode_dict_time - reported.decode_dict_time); + } + if (_profile.decode_level_time != nullptr) { + COUNTER_UPDATE(_profile.decode_level_time, + stats.decode_level_time - reported.decode_level_time); + } + if (_profile.decode_null_map_time != nullptr) { + COUNTER_UPDATE(_profile.decode_null_map_time, + stats.decode_null_map_time - reported.decode_null_map_time); + } + if (_profile.materialization_time != nullptr) { + COUNTER_UPDATE(_profile.materialization_time, + stats.materialization_time - reported.materialization_time); + } + if (_profile.hybrid_selection_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_batches, + stats.hybrid_selection_batches - reported.hybrid_selection_batches); + } + if (_profile.hybrid_selection_ranges != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_ranges, + stats.hybrid_selection_ranges - reported.hybrid_selection_ranges); + } + if (_profile.hybrid_selection_null_fallback_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_null_fallback_batches, + stats.hybrid_selection_null_fallback_batches - + reported.hybrid_selection_null_fallback_batches); + } + if (_profile.page_index_read_calls != nullptr) { + COUNTER_UPDATE(_profile.page_index_read_calls, + stats.page_index_read_calls - reported.page_index_read_calls); + } + if (_profile.skip_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.skip_page_header_count, + stats.skip_page_header_num - reported.skip_page_header_num); + } + if (_profile.parse_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.parse_page_header_count, + stats.parse_page_header_num - reported.parse_page_header_num); + } + if (_profile.read_page_header_time != nullptr) { + COUNTER_UPDATE(_profile.read_page_header_time, + stats.read_page_header_time - reported.read_page_header_time); + } + const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; + if (_profile.page_read_count != nullptr) { + COUNTER_UPDATE(_profile.page_read_count, page_read_delta); + } + if (_profile.page_cache_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_write_count, + stats.page_cache_write_counter - reported.page_cache_write_counter); + } + if (_profile.page_cache_compressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_write_count, + stats.page_cache_compressed_write_counter - + reported.page_cache_compressed_write_counter); + } + if (_profile.page_cache_decompressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_write_count, + stats.page_cache_decompressed_write_counter - + reported.page_cache_decompressed_write_counter); + } + if (_profile.page_cache_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_hit_count, + stats.page_cache_hit_counter - reported.page_cache_hit_counter); + } + if (_profile.page_cache_miss_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_miss_count, + stats.page_cache_missing_counter - reported.page_cache_missing_counter); + } + if (_profile.page_cache_compressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, + stats.page_cache_compressed_hit_counter - + reported.page_cache_compressed_hit_counter); + } + if (_profile.page_cache_decompressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, + stats.page_cache_decompressed_hit_counter - + reported.page_cache_decompressed_hit_counter); + } + _reported_native_stats = stats; + return page_read_delta; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h new file mode 100644 index 00000000000000..0ffb8d054008b7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format_v2/column_data.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/native/column_reader.h" + +namespace doris { +class RuntimeState; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { + +class NativeParquetMetadata; + +namespace detail { +inline constexpr int64_t MAX_NATIVE_LAZY_SKIP_ROWS = std::numeric_limits::max(); + +inline int64_t bounded_native_lazy_skip_rows(int64_t rows) { + return std::min(rows, MAX_NATIVE_LAZY_SKIP_ROWS); +} +} // namespace detail + +// Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native +// Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final +// Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested +// values_column. +// +// Cursor contract: +// - read(rows) consumes and appends exactly `rows` logical top-level rows; +// - select(selection, batch_rows) consumes `batch_rows` and appends only selected rows; +// - skip(rows) consumes `rows` with an all-false FilterMap and appends no payload; +// - one adapter lives for one top-level column in one Row Group, so decoder dictionaries, +// decompression buffers, level buffers, converters, and destination capacity survive adaptive +// batch-size changes. +class NativeColumnReader final : public ParquetColumnReader { +public: + static Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, io::FileReaderSPtr file, + const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, + RuntimeState* runtime_state, bool enable_page_cache, + const std::string& page_cache_file_key, bool enable_dictionary_filter, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + ~NativeColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column) override; + Status select_with_dictionary_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + Status select_with_fixed_width_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, const VExprSPtrs& conjuncts, + int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) override; + void flush_profile() override; + bool crossed_page_since_last_batch() override; + Result dictionary_values() override; + +private: + NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, + ParquetColumnReaderProfile profile); + + Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter); + + Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + MutableColumnPtr& column, const DataTypePtr& output_type, + bool dictionary_ids, int64_t* rows_read); + Status read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); + Status validate_selected_span(int64_t rows); + void advance_selected_span(int64_t rows); + + // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. + segment_v2::RowRanges _row_ranges; + std::set _projected_column_ids; + std::set _filter_column_ids; + const std::unordered_map* _offset_indexes = nullptr; + std::shared_ptr _schema_node; + std::unique_ptr _native_reader; + std::unique_ptr _page_cache_runtime_state; + std::vector _selected_ranges; + size_t _selected_range_idx = 0; + int64_t _logical_row_position = 0; + int64_t _row_group_rows = 0; + + bool _dictionary_filter_enabled = false; + bool _nested = false; + // The native tree exposes cumulative statistics. Keep the last reported snapshot so each + // FileScannerV2 batch contributes only its delta to RuntimeProfile. + native::ColumnReader::ColumnStatistics _reported_native_stats; + // Page-crossing is sampled at every scheduler batch, independently of amortized profile flushes. + std::vector _batch_leaf_page_read_counters; + std::vector _filter_scratch; + size_t _batches_since_scratch_check = 0; + MutableColumnPtr _skip_column; + MutableColumnPtr _dictionary_id_column; + MutableColumnPtr _matched_dictionary_ids; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp deleted file mode 100644 index e06b7eaaf317e7..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/nested_column_materializer.h" - -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" - -namespace doris::format::parquet { - -ColumnArray* array_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnMap* map_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return &nullable_column->get_null_map_data(); - } - return nullptr; -} - -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { - offsets.reserve(offsets.size() + entry_counts.size()); - uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); - for (const auto entry_count : entry_counts) { - current_offset += entry_count; - offsets.push_back(current_offset); - } -} - -void append_parent_nulls(NullMap* dst, const NullMap& src) { - if (dst == nullptr) { - return; // target column is not nullable; no null marker is needed - } - dst->insert(src.begin(), src.end()); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h deleted file mode 100644 index 90fac01eb2f5e5..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_struct.h" - -namespace doris::format::parquet { - -// ============================================================================ -// ============================================================================ - -ColumnArray* array_column_from_output(MutableColumnPtr& column); - -ColumnMap* map_column_from_output(MutableColumnPtr& column); - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column); - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column); - -// offsets[i] = offsets[i-1] + entry_counts[i]. -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); - -void append_parent_nulls(NullMap* dst, const NullMap& src); - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp deleted file mode 100644 index fd261ef5219d27..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ /dev/null @@ -1,803 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "runtime/runtime_profile.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { - switch (time_unit) { - case ParquetTimeUnit::MILLIS: - return DecodedTimeUnit::MILLIS; - case ParquetTimeUnit::MICROS: - return DecodedTimeUnit::MICROS; - case ParquetTimeUnit::NANOS: - return DecodedTimeUnit::NANOS; - case ParquetTimeUnit::UNKNOWN: - default: - return DecodedTimeUnit::UNKNOWN; - } -} - -Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, - size_t* value_size) { - switch (value_kind) { - case DecodedValueKind::BOOL: - *value_size = sizeof(bool); - return Status::OK(); - case DecodedValueKind::INT32: - *value_size = sizeof(int32_t); - return Status::OK(); - case DecodedValueKind::UINT32: - *value_size = sizeof(uint32_t); - return Status::OK(); - case DecodedValueKind::INT64: - *value_size = sizeof(int64_t); - return Status::OK(); - case DecodedValueKind::UINT64: - *value_size = sizeof(uint64_t); - return Status::OK(); - case DecodedValueKind::INT96: - *value_size = 12; - return Status::OK(); - case DecodedValueKind::FLOAT: - *value_size = sizeof(float); - return Status::OK(); - case DecodedValueKind::DOUBLE: - *value_size = sizeof(double); - return Status::OK(); - case DecodedValueKind::BINARY: - case DecodedValueKind::FIXED_BINARY: - return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", - column_name); - } - return Status::InternalError("Unknown decoded value kind for column {}", column_name); -} - -Status get_binary_chunks(const std::string& column_name, - ::parquet::internal::RecordReader& record_reader, - std::vector>* chunks) { - if (auto* dictionary_reader = - dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); - dictionary_reader != nullptr) { - auto chunked = dictionary_reader->GetResult(); - if (chunked == nullptr) { - return Status::Corruption( - "Parquet dictionary record reader returned null result for column {}", - column_name); - } - *chunks = chunked->chunks(); - return Status::OK(); - } - auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); - if (binary_reader == nullptr) { - return Status::InternalError("Parquet binary record reader is not available for column {}", - column_name); - } - *chunks = binary_reader->GetBuilderChunks(); - return Status::OK(); -} - -Status append_dictionary_binary_values(const std::string& column_name, - const ::arrow::DictionaryArray& dictionary_array, - std::vector* values) { - DORIS_CHECK(values != nullptr); - const auto& dictionary = dictionary_array.dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - column_name); - } - auto append_value = [&](int64_t dictionary_index) -> Status { - if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back( - reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); - }; - for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { - if (dictionary_array.IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); - } - return Status::OK(); -} - -Status build_binary_values(const std::string& column_name, - const std::vector>& chunks, - int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values) { - std::vector compact_values; - auto* values = read_dense_for_nullable ? &compact_values : binary_values; - values->reserve(records_read); - for (const auto& chunk : chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { - if (binary_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(row_idx, &length); - values->emplace_back(reinterpret_cast(value), length); - } - } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { - if (fixed_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), - fixed_array->byte_width()); - } - } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { - RETURN_IF_ERROR( - append_dictionary_binary_values(column_name, *dictionary_array, values)); - } else { - return Status::InternalError("Unexpected Arrow binary array type for column {}", - column_name); - } - } - if (read_dense_for_nullable) { - if (null_map == nullptr || null_map->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", - column_name, records_read, null_map == nullptr ? 0 : null_map->size()); - } - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values.size() != static_cast(non_null_count)) { - return Status::Corruption( - "Invalid dense nullable parquet binary values for column {}: values={}, " - "records={}, nulls={}", - column_name, compact_values.size(), records_read, - records_read - non_null_count); - } - binary_values->reserve(records_read); - size_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - binary_values->emplace_back(static_cast(nullptr), 0); - continue; - } - binary_values->emplace_back(compact_values[value_idx++]); - } - return Status::OK(); - } - if (binary_values->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid parquet binary record read result for column {}: rows={}, records={}", - column_name, binary_values->size(), records_read); - } - return Status::OK(); -} - -float half_to_float(uint16_t value) { - const uint32_t sign = (value & 0x8000U) << 16; - const uint32_t exponent = (value & 0x7C00U) >> 10; - const uint32_t mantissa = value & 0x03FFU; - - if (exponent == 0) { - if (mantissa == 0) { - return std::bit_cast(sign); - } - const float subnormal = std::ldexp(static_cast(mantissa), -24); - return sign == 0 ? subnormal : -subnormal; - } - if (exponent == 0x1FU) { - return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); - } - return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); -} - -Status build_float16_values(const std::string& column_name, - const ParquetTypeDescriptor& type_descriptor, - const std::vector& binary_values, int64_t row_count, - std::vector* float_values) { - if (type_descriptor.fixed_length != 2) { - return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, - type_descriptor.fixed_length); - } - if (binary_values.size() != static_cast(row_count)) { - return Status::Corruption( - "Invalid parquet Float16 value count for column {}: values={}, rows={}", - column_name, binary_values.size(), row_count); - } - float_values->resize(static_cast(row_count)); - for (int64_t row = 0; row < row_count; ++row) { - const auto& binary_value = binary_values[static_cast(row)]; - if (binary_value.data == nullptr && binary_value.size == 0) { - (*float_values)[static_cast(row)] = 0; - continue; - } - if (binary_value.data == nullptr || binary_value.size != 2) { - return Status::Corruption( - "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", - column_name, row, binary_value.data == nullptr ? "null" : "non-null", - binary_value.size); - } - uint16_t raw_value = 0; - std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); - (*float_values)[static_cast(row)] = half_to_float(raw_value); - } - return Status::OK(); -} - -} // namespace - -Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - batch->_values_written = record_reader.values_written(); - - if (!batch->is_binary_value()) { - batch->_fixed_values = record_reader.values(); - return Status::OK(); - } - - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); - batch->_values_written = 0; - for (const auto& chunk : batch->_binary_chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", _name); - } - batch->_values_written += chunk->length(); - } - return Status::OK(); -} - -Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - - // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() - // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be - // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read - // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then - // reading ["c"] would report one current level but three values. Release the chunks here and - // let the temporary vector destroy them immediately. We deliberately do not inspect or copy - // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value - // materialization. - if (batch->is_binary_value()) { - std::vector> discarded_chunks; - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); - } - - // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the - // RecordReader and are cleared by Reset(); binary values were released above solely to reset - // the Arrow builder. - batch->_values_written = 0; - return Status::OK(); -} - -// - FLOAT16: binary -> half_to_float -> float_values -Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, MutableColumnPtr& column) const { - std::vector binary_values; - std::vector spaced_values; - std::vector float_values; - DecodedColumnView view; - view.value_kind = batch._value_kind; - view.time_unit = decoded_time_unit(_type_descriptor.time_unit); - view.row_count = row_count; - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.decimal_precision = _type_descriptor.decimal_precision; - view.decimal_scale = _type_descriptor.decimal_scale; - view.fixed_length = _type_descriptor.fixed_length; - view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; - view.timezone = _timezone; - view.enable_strict_mode = _enable_strict_mode; - view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); - const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; - - if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, - &float_values)); - view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(float_values.data()); - } else if (batch.is_binary_value()) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - view.binary_values = &binary_values; - } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); - view.values = spaced_values.data(); - } else { - view.values = batch._fixed_values; - } - - if (_decoded_value_appender != nullptr) { - return _decoded_value_appender(column, view); - } - - { - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& tmp_null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = tmp_null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(tmp_null_map.data() + old_null_map_size, 0, - tmp_null_map.size() - old_null_map_size); - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } - return Status::OK(); -} - -bool ParquetLeafBatch::is_binary_value() const { - return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; -} - -Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, - int64_t row_count, const NullMap* null_map, - std::vector* spaced_values) const { - DORIS_CHECK(null_map != nullptr); - DORIS_CHECK(spaced_values != nullptr); - size_t value_size = 0; - RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); - spaced_values->resize(static_cast(row_count) * value_size); - const auto non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (batch._values_written != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet values for column {}: values={}, records={}, " - "nulls={}", - _name, batch._values_written, row_count, row_count - non_null_count); - } - auto* dst = spaced_values->data(); - int64_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - continue; // NULL row: skip it and keep the target slot zeroed - } - std::memcpy(dst + static_cast(record_idx) * value_size, - batch._fixed_values + static_cast(value_idx) * value_size, value_size); - ++value_idx; - } - return Status::OK(); -} - -ParquetLeafReader::ParquetLeafReader( - const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, - DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, - bool enable_strict_mode, - std::function decoded_value_appender) - : _descriptor(descriptor), - _type_descriptor(type_descriptor), - _type(std::move(type)), - _name(std::move(name)), - _record_reader(std::move(record_reader)), - _profile(profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)) {} - -Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, - int64_t* rows_read) const { - if (batch == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet records for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > batch_rows) { - return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, - *rows_read); - } - return collect_batch(*_record_reader, batch); -} - -Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const { - if (_descriptor->max_definition_level() == 0) { - return Status::OK(); - } - auto* def_levels = batch.def_levels(); - if (def_levels == nullptr && records_read > 0) { - return Status::Corruption( - "Parquet record reader returned null definition levels for nullable column {}", - _name); - } - const int16_t max_definition_level = _descriptor->max_definition_level(); - null_map->resize(records_read); - auto* __restrict dst = null_map->data(); - const auto* __restrict src = def_levels; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - dst[record_idx] = src[record_idx] != max_definition_level; - } - return Status::OK(); -} - -Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - ParquetLeafBatch leaf_batch; - int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, - batch, value_slot_repetition_level); -} - -Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - int64_t records_read = 0; - ParquetLeafBatch leaf_batch; - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - records_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } - if (records_read < 0 || records_read > batch_rows) { - return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, - records_read); - } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); -} - -Status ParquetLeafReader::build_nested_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->value_slot_definition_level = value_slot_definition_level; - batch->value_slot_repetition_level = value_slot_repetition_level; - - batch->records_read = records_read; - if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { - return Status::NotSupported( - "Dense nullable parquet nested reader is not supported for column {}", _name); - } - batch->levels_written = leaf_batch.consumed_level_count(); - const int64_t values_written = leaf_batch.values_written(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - if (batch->levels_written == 0 && batch->records_read > 0 && - values_written == batch->records_read && _descriptor->max_definition_level() == 0 && - _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read || values_written < 0 || - values_written > batch->levels_written) { - return Status::Corruption( - "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", - _name, batch->records_read, batch->levels_written, values_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - - const int16_t leaf_definition_level = _descriptor->max_definition_level(); - // Arrow's RecordReader may emit value placeholders for null ancestors that are below the - // Doris materialization threshold. Those slots must still advance the payload value index; - // otherwise the next defined child level points at the placeholder instead of its real value. - auto count_value_slots = [&](int16_t slot_definition_level) { - int64_t slot_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] >= slot_definition_level && - batch->rep_levels[level_idx] <= value_slot_repetition_level) { - ++slot_count; - } - } - return slot_count; - }; - - const int64_t value_slot_count = count_value_slots(value_slot_definition_level); - int16_t payload_slot_definition_level = value_slot_definition_level; - int64_t payload_value_slot_count = value_slot_count; - while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { - --payload_slot_definition_level; - payload_value_slot_count = count_value_slots(payload_slot_definition_level); - } - - int64_t leaf_value_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < value_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - if (batch->def_levels[level_idx] == leaf_definition_level) { - ++leaf_value_count; - } - } - - enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; - ValueLayout value_layout = ValueLayout::LEAF_VALUES; - if (values_written == batch->levels_written) { - value_layout = ValueLayout::LEVELS; - } else if (values_written == value_slot_count) { - value_layout = ValueLayout::VALUE_SLOTS; - } else if (values_written == leaf_value_count) { - value_layout = ValueLayout::LEAF_VALUES; - } else if (values_written == payload_value_slot_count) { - value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; - } else { - return Status::Corruption( - "Nested parquet reader returned inconsistent value count for column {}: values={}, " - "levels={}, slots={}, leaf_values={}, payload_slots={}, " - "payload_slot_definition_level={}", - _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, - payload_value_slot_count, payload_slot_definition_level); - } - - batch->value_indices.resize(static_cast(batch->levels_written), -1); - NullMap value_nulls(static_cast(values_written), 1); - int64_t value_idx = 0; - const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS - ? payload_slot_definition_level - : value_slot_definition_level; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < decoded_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; - int64_t decoded_value_idx = -1; - if (value_layout == ValueLayout::LEVELS) { - decoded_value_idx = level_idx; - } else if (value_layout == ValueLayout::VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else { - if (!has_leaf_value) { - continue; - } - decoded_value_idx = value_idx++; - } - DORIS_CHECK(decoded_value_idx >= 0); - DORIS_CHECK(decoded_value_idx < values_written); - if (has_leaf_value) { - batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - value_nulls[static_cast(decoded_value_idx)] = 0; - } - } - if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { - return Status::Corruption( - "Nested parquet reader value cursor stopped early for column {}: values={}, " - "visited={}", - _name, values_written, value_idx); - } - - const auto value_type = remove_nullable(_type); - batch->values_column = value_type->create_column(); - if (values_written > 0) { - ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, - _record_reader, _profile, _timezone, _enable_strict_mode); - RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, - batch->values_column)); - } - return Status::OK(); -} - -Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->records_read = records_read; - batch->levels_written = leaf_batch.consumed_level_count(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - - // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot - // per top-level row so the COUNT(col) aggregation code can use the same shape loop. - if (batch->levels_written == 0 && batch->records_read > 0 && - _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read) { - return Status::Corruption( - "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, - batch->records_read, batch->levels_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - return Status::OK(); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h deleted file mode 100644 index b396b35fd1f32c..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace arrow { -class Array; -} // namespace arrow - -namespace doris::format::parquet { - -struct ParquetLeafReaderTestAccess; - -// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. -// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. -// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. -struct ParquetNestedScalarBatch { - int64_t records_read = 0; - int64_t levels_written = 0; - int16_t value_slot_definition_level = 0; - int16_t value_slot_repetition_level = std::numeric_limits::max(); - std::vector def_levels; - std::vector rep_levels; - std::vector value_indices; - MutableColumnPtr values_column; - - bool empty() const { return levels_written == 0; } -}; - -class ParquetLeafBatch { -public: - int64_t consumed_level_count() const { return _consumed_level_count; } - int64_t decoded_level_count() const { return _decoded_level_count; } - int64_t values_written() const { return _values_written; } - bool read_dense_for_nullable() const { return _read_dense_for_nullable; } - const int16_t* def_levels() const { return _def_levels; } - const int16_t* rep_levels() const { return _rep_levels; } - const std::vector>& binary_chunks() const { - return _binary_chunks; - } - -private: - friend class ParquetLeafReader; - - bool is_binary_value() const; - - DecodedValueKind _value_kind = DecodedValueKind::INT32; - int64_t _consumed_level_count = 0; - int64_t _decoded_level_count = 0; - int64_t _values_written = 0; - const int16_t* _def_levels = nullptr; - const int16_t* _rep_levels = nullptr; - const uint8_t* _fixed_values = nullptr; - bool _read_dense_for_nullable = false; - std::vector> _binary_chunks; -}; - -// read_batch() -> build_null_map() + append_values() -// read_nested_batch() -class ParquetLeafReader { -public: - ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, - ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile = {}, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - std::function - decoded_value_appender = nullptr); - - Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; - - Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const; - - Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, - MutableColumnPtr& column) const; - - // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. - Status read_nested_batch( - int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level = std::numeric_limits::max()) const; - - // COUNT(col) and nested-skip shape-only read path. It still calls Arrow - // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but - // Doris only copies levels: - // - it does not build value_indices or values_column - // - it does not enter DataTypeSerde::read_column_from_decoded_values() - // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is - // the RecordReader's required reset operation; it never copies them into a Doris Column - // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance - // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not - // expose a public levels-only API, so ReadRecords may still perform required page decoding. - Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; - -private: - friend struct ParquetLeafReaderTestAccess; - - Status collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not - // expose value buffers. Binary chunks are released only to reset Arrow's builder and are - // immediately discarded. Used by COUNT(col) and nested skip. - Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - std::vector* spaced_values) const; - - Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const; - Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - ParquetNestedScalarBatch* batch) const; - - const ::parquet::ColumnDescriptor* _descriptor = - nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) - ParquetTypeDescriptor - _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) - DataTypePtr _type; // Doris target type - std::string _name; // column name for error messages - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader (shared ownership) - ParquetColumnReaderProfile _profile; // profile counters - const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion - bool _enable_strict_mode = false; // strict mode for type mismatch errors - std::function _decoded_value_appender; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp index 4e9a363b13c7cb..dcd601b003ae3c 100644 --- a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -20,6 +20,7 @@ #include "core/assert_cast.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp deleted file mode 100644 index 520a3cb1e62e3b..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ /dev/null @@ -1,584 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/scalar_column_reader.h" - -#include -#include -#include - -#include -#include -#include - -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -class ParquetNestedScalarValueCursor { -public: - explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } - - void reset(const ParquetNestedScalarBatch* batch) { - DORIS_CHECK(batch != nullptr); - _batch = batch; - } - - Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { - DORIS_CHECK(_batch != nullptr); - DORIS_CHECK(value_idx != nullptr); - DORIS_CHECK(level_idx < _batch->levels_written); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); - const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; - if (computed_value_idx < 0) { - return Status::Corruption("Nested parquet value is absent for column {}", column_name); - } - DORIS_CHECK(_batch->values_column.get() != nullptr); - if (computed_value_idx >= _batch->values_column->size()) { - return Status::Corruption("Nested parquet value index is out of range for column {}", - column_name); - } - *value_idx = computed_value_idx; - return Status::OK(); - } - -private: - const ParquetNestedScalarBatch* _batch = nullptr; -}; - -Status append_scalar_batch_value(const ScalarColumnReader& column_reader, - const ParquetNestedScalarBatch& batch, int64_t level_idx, - ParquetNestedScalarValueCursor* value_cursor, - MutableColumnPtr& column) { - DORIS_CHECK(value_cursor != nullptr); - int64_t value_idx = -1; - RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); - auto* nullable_column = check_and_get_column(*column); - if (nullable_column != nullptr) { - nullable_column->get_nested_column().insert_from(*batch.values_column, - static_cast(value_idx)); - nullable_column->get_null_map_data().push_back(0); - return Status::OK(); - } - column->insert_from(*batch.values_column, static_cast(value_idx)); - return Status::OK(); -} - -Status append_arrow_binary_dictionary_value(const std::string& column_name, - const ::arrow::Array& dictionary, - int64_t dictionary_index, - std::vector* values) { - DORIS_CHECK(values != nullptr); - if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast(&dictionary)) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast(&dictionary)) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); -} - -} // namespace - -ScalarColumnReader::ScalarColumnReader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile profile) - : ParquetColumnReader(column_schema, column_schema.type, profile), - _descriptor(column_schema.descriptor), - _type_descriptor(column_schema.type_descriptor), - _record_reader(std::move(record_reader)), - _page_skip_plan(page_skip_plan), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _nested_batch(std::make_unique()) {} - -ScalarColumnReader::~ScalarColumnReader() = default; - -Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - auto reader = leaf_reader(); - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); - - NullMap null_map; - RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); - const auto value_kind = decoded_value_kind(_type_descriptor); - const bool is_binary_value = - value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map.data()), null_map.size())); - const int64_t null_count = *rows_read - non_null_count; - if (leaf_batch.values_written() != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet record read result for column {}: values={}, " - "records={}, nulls={}", - _name, leaf_batch.values_written(), *rows_read, null_count); - } - } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && - leaf_batch.values_written() != *rows_read) { - return Status::Corruption( - "Invalid parquet record read result for column {}: values={}, records={}", _name, - leaf_batch.values_written(), *rows_read); - } - - RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::skip_records(int64_t rows) { - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - if (rows <= 0) { - return Status::OK(); - } - int64_t skipped_rows = 0; - SCOPED_TIMER(_profile.arrow_skip_records_time); - try { - _record_reader->Reset(); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Failed to skip parquet records for column {}: skipped {} of {} rows", - _name, skipped_rows, rows); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { - if (_page_skip_plan == nullptr || rows <= 0) { - return 0; - } - const int64_t skip_end = _row_group_rows_read + rows; - int64_t filtered_rows = 0; - for (const auto& range : _page_skip_plan->skipped_ranges) { - const int64_t range_end = range.start + range.length; - if (range_end <= _row_group_rows_read) { - continue; - } - if (range.start >= skip_end) { - break; - } - const int64_t start = std::max(range.start, _row_group_rows_read); - const int64_t end = std::min(range_end, skip_end); - if (start < end) { - // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered - // range can only overlap such a gap when the whole data page is outside every selected - // range, so partial overlap would mean the planner and scheduler are out of sync. - DORIS_CHECK(start == range.start); - DORIS_CHECK(end == range_end); - filtered_rows += end - start; - } - } - return filtered_rows; -} - -void ScalarColumnReader::advance_rows_read(int64_t rows) { - DORIS_CHECK(rows >= 0); - _row_group_rows_read += rows; -} - -Status ScalarColumnReader::skip(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); - DORIS_CHECK(page_filtered_rows <= rows); - const int64_t record_reader_skip_rows = rows - page_filtered_rows; - RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); - advance_rows_read(rows); - return Status::OK(); -} - -Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, - bool* used_filter) { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); - *used_filter = false; - row_filter->clear(); - row_filter->reserve(selected_rows); - DORIS_CHECK(_record_reader != nullptr); - // A clean fallback is possible only before any range skip or read advances the record reader. - // Once dictionary selection starts, losing dictionary output is corruption rather than a - // reason to retry through the ordinary selected-read path with an already advanced stream. - if (!_record_reader->read_dictionary()) { - return Status::OK(); - } - *used_filter = true; - - const auto ranges = selection_to_ranges(sel, selected_rows); - int64_t cursor = 0; - for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument( - "Invalid parquet dictionary selection range [{}, {}) for column {}", - range.start, range.start + range.length, _name); - } - RETURN_IF_ERROR(skip(range.start - cursor)); - - int64_t range_rows_read = 0; - RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, - row_filter, &range_rows_read, - used_filter)); - if (!*used_filter) { - return Status::OK(); - } - if (range_rows_read != range.length) { - return Status::Corruption( - "Parquet dictionary selected read returned {} rows, expected {} rows for " - "column {}", - range_rows_read, range.length, _name); - } - cursor = range.start + range.length; - } - RETURN_IF_ERROR(skip(batch_rows - cursor)); - if (_profile.reader_select_rows != nullptr) { - COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); - } - return Status::OK(); -} - -Status ScalarColumnReader::read_range_with_dictionary_filter( - int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(rows_read != nullptr); - DORIS_CHECK(used_filter != nullptr); - DORIS_CHECK(_record_reader != nullptr); - if (!_record_reader->read_dictionary()) { - return Status::Corruption( - "Parquet dictionary reader became unavailable after selected reading started for " - "column {}", - _name); - } - - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); - int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, - column, row_filter, &matched_rows, - used_filter)); - if (!*used_filter) { - return Status::Corruption( - "Parquet dictionary reader did not return dictionary batches for column {}", _name); - } - if (row_filter->size() < static_cast(*rows_read)) { - return Status::Corruption( - "Parquet dictionary filter produced too few row decisions for column {}: " - "filter={}, rows={}", - _name, row_filter->size(), *rows_read); - } - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(matched_rows != nullptr); - DORIS_CHECK(used_filter != nullptr); - *matched_rows = 0; - *used_filter = false; - - std::vector selected_values; - for (const auto& chunk : chunks) { - DORIS_CHECK(chunk != nullptr); - const auto* dict_array = dynamic_cast(chunk.get()); - if (dict_array == nullptr) { - // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a - // normal selected read would desynchronize the Parquet stream, so absence of a - // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). - return Status::OK(); - } - *used_filter = true; - const auto& dictionary = dict_array->dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - _name); - } - - // Dictionary predicates are evaluated once against the dictionary page and produce a - // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows - // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. - for (int64_t row = 0; row < dict_array->length(); ++row) { - bool keep = false; - if (!dict_array->IsNull(row)) { - const int64_t dictionary_index = dict_array->GetValueIndex(row); - if (dictionary_index < 0 || dictionary_index >= dictionary->length() || - dictionary_index >= static_cast(dictionary_filter.size())) { - return Status::Corruption( - "Invalid parquet dictionary index {} for column {}: dictionary={}, " - "filter={}", - dictionary_index, _name, dictionary->length(), - dictionary_filter.size()); - } - keep = dictionary_filter[static_cast(dictionary_index)] != 0; - if (keep) { - RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &selected_values)); - ++*matched_rows; - } - } - row_filter->push_back(keep ? 1 : 0); - } - } - - if (!*used_filter) { - return Status::OK(); - } - return append_decoded_binary_values(selected_values, column); -} - -Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const { - DecodedColumnView view; - view.value_kind = decoded_value_kind(_type_descriptor); - view.row_count = static_cast(values.size()); - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.fixed_length = _type_descriptor.fixed_length; - view.binary_values = &values; - - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); - return Status::OK(); - } - return _type->get_serde()->read_column_from_decoded_values(*column, view); - } - - NullMap null_map(values.size(), 0); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - return _type->get_serde()->read_column_from_decoded_values(*column, view); -} - -// The value index stream must advance on those null slots, otherwise later payload values shift. -Status ScalarColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, - _nested_batch.get(), _repetition_level)); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - // The levels-only loader intentionally does not populate value-slot metadata or payload - // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() - // does, so this consumer works for both loaded batch forms. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_read = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - if (def_level == _definition_level) { - RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, - &value_cursor, column)); - } else { - if (!_type->is_nullable() && def_level >= _nullable_definition_level) { - return Status::Corruption( - "Parquet scalar column {} contains null for non-nullable field", _name); - } - column->insert_default(); - } - ++*values_read; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - if (values_consumed == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical - // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition - // level below a non-null value, while a required leaf owns a slot only at its full definition - // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_consumed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); - ++*values_consumed; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); - } - if (!_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", - _name); - } - column->insert_default(); - return Status::OK(); -} - -Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return Status::OK(); - } - if (require_non_null || !_type->is_nullable()) { - return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", - _name); - } - return Status::OK(); -} - -const std::vector& ScalarColumnReader::nested_definition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->def_levels; -} - -const std::vector& ScalarColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->rep_levels; -} - -int64_t ScalarColumnReader::nested_levels_written() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->levels_written; -} - -bool ScalarColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h deleted file mode 100644 index 5342baa803eca0..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_type.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris::format::parquet { - -struct ScalarColumnReaderTestAccess; - -// load_nested_batch() / build_nested_column() -class ScalarColumnReader final : public ParquetColumnReader { - friend class MapColumnReader; - friend struct ScalarColumnReaderTestAccess; - -public: - ScalarColumnReader(const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan = nullptr, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - ParquetColumnReaderProfile profile = {}); - ~ScalarColumnReader() override; - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - bool* used_filter) override; - - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - -private: - Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; - Status validate_nested_value(int64_t level_idx, bool require_non_null) const; - Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter); - Status append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; - Status append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const; - - const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - - ParquetLeafReader leaf_reader() const { - return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, - _profile, _timezone, _enable_strict_mode); - } - - void advance_rows_read(int64_t rows); - Status skip_records(int64_t rows); - int64_t page_filtered_rows_to_skip(int64_t rows) const; - - const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor - ParquetTypeDescriptor _type_descriptor; // type encoding information - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader - const ParquetPageSkipPlan* _page_skip_plan = - nullptr; // page-index pruning result (may be nullptr) - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) - std::unique_ptr _nested_batch; // intermediate result for nested reads -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp deleted file mode 100644 index 5abe7abe75e9a2..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/struct_column_reader.h" - -#include -#include -#include -#include - -#include "core/column/column_struct.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { - -ParquetColumnReader* StructColumnReader::shape_source_reader() const { - for (const auto& child : _children) { - auto* child_reader = child.get(); - DORIS_CHECK(child_reader != nullptr); - if (!child_reader->is_or_has_repeated_child()) { - return child_reader; - } - } - if (_children.empty()) { - return nullptr; - } - return _children[0].get(); -} - -Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const { - DORIS_CHECK(child_reader != nullptr); - const int64_t next_child_cursor = parent_level_idx + 1; - if (auto* scalar_child = dynamic_cast(child_reader)) { - if (next_child_cursor > scalar_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - scalar_child->name(), _name); - } - scalar_child->set_nested_build_level_cursor( - std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); - return Status::OK(); - } - if (auto* struct_child = dynamic_cast(child_reader); - struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { - if (next_child_cursor > struct_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - struct_child->name(), _name); - } - struct_child->set_nested_build_level_cursor( - std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); - for (auto& grandchild : struct_child->_children) { - RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), - parent_level_idx)); - } - return Status::OK(); - } - - int64_t child_cursor = child_reader->nested_build_level_cursor(); - const auto& child_rep_levels = child_reader->nested_repetition_levels(); - const int64_t child_levels_written = child_reader->nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { - break; - } - } - child_reader->set_nested_build_level_cursor(child_cursor); - return Status::OK(); -} - -Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status StructColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status StructColumnReader::load_nested_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::load_nested_levels_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument( - "Invalid parquet struct process result pointer for column {}", _name); - } - if (_children.empty()) { - if (column != nullptr) { - (*column)->resize((*column)->size() + static_cast(length_upper_bound)); - } - *values_processed = length_upper_bound; - return Status::OK(); - } - ColumnStruct* struct_column = nullptr; - NullMap* parent_null_map = nullptr; - if (column != nullptr) { - struct_column = struct_column_from_output(*column); - DORIS_CHECK(struct_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - } - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - const auto& def_levels = shape_reader->nested_definition_levels(); - const auto& rep_levels = shape_reader->nested_repetition_levels(); - const int64_t levels_written = shape_reader->nested_levels_written(); - - NullMap parent_nulls; - std::vector parent_level_indices; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < levels_written) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = - !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (def_level < _repeated_ancestor_definition_level) { - continue; - } - if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { - continue; - } - const bool parent_is_null = def_level < _nullable_definition_level; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption( - "Parquet STRUCT column {} contains null for non-nullable struct", _name); - } - parent_nulls.push_back(parent_is_null); - parent_level_indices.push_back(current_level_idx); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - std::vector child_columns; - if (column != nullptr) { - child_columns.reserve(struct_column->get_columns().size()); - for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); - } - } - for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { - const int output_idx = _child_output_indices[child_idx]; - if (column != nullptr && output_idx < 0) { - continue; - } - // STRUCT owns row alignment. Child readers consume only present parent rows from their - // level streams; null STRUCT parents become default placeholders in every child column. - // This mirrors Arrow's separation between struct validity and child array materialization, - // and avoids asking scalar/list/map children to invent values for an absent parent. - int64_t pending_present_rows = 0; - int64_t total_child_rows = 0; - auto flush_present_rows = [&]() -> Status { - if (pending_present_rows == 0) { - return Status::OK(); - } - int64_t child_rows = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, child_columns[output_idx], &child_rows)); - } else { - RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, - &child_rows)); - } - if (child_rows != pending_present_rows) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), child_rows, pending_present_rows, _name); - } - total_child_rows += child_rows; - pending_present_rows = 0; - return Status::OK(); - }; - for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = parent_nulls[parent_idx]; - if (!parent_is_null) { - ++pending_present_rows; - continue; - } - RETURN_IF_ERROR(flush_present_rows()); - if (column != nullptr) { - child_columns[output_idx]->insert_default(); - } - RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - parent_level_indices[parent_idx])); - ++total_child_rows; - } - RETURN_IF_ERROR(flush_present_rows()); - if (total_child_rows != *values_processed) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), total_child_rows, *values_processed, _name); - } - } - if (column != nullptr) { - for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); - } - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& StructColumnReader::nested_definition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_definition_levels(); -} - -const std::vector& StructColumnReader::nested_repetition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_repetition_levels(); -} - -int64_t StructColumnReader::nested_levels_written() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_levels_written(); -} - -bool StructColumnReader::is_or_has_repeated_child() const { - auto* shape_reader = shape_source_reader(); - return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); -} - -void StructColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - for (auto& child : _children) { - DORIS_CHECK(child != nullptr); - child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - } -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h deleted file mode 100644 index 3c2d6904cb36f4..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class StructColumnReader final : public ParquetColumnReader { -public: - StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::vector> children, - std::vector child_output_indices, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _children(std::move(children)), - _child_output_indices(std::move(child_output_indices)) { - DCHECK_EQ(_children.size(), _child_output_indices.size()); - } - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - ParquetColumnReader* shape_source_reader() const; - Status advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const; - - std::vector> _children; // projected child readers - std::vector _child_output_indices; // child reader -> struct output position mapping -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 589154d4acc0e4..ab2bf93c785edc 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -33,11 +34,11 @@ struct RowRange { struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes - // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + // dictionary pages, matching the native page reader's ordinal domain. std::vector skipped_pages; std::vector skipped_page_compressed_sizes; - // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid - // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + // Row ranges covered by skipped data pages. NativeColumnReader uses these ranges to avoid + // consuming logical rows twice after the page reader has already skipped their payload. std::vector skipped_ranges; bool empty() const { return skipped_ranges.empty(); } @@ -66,6 +67,7 @@ class SelectionVector { _owned.clear(); _data = data; _size = count; + ++_generation; } void resize(size_t count) { @@ -75,12 +77,14 @@ class SelectionVector { for (size_t idx = 0; idx < count; ++idx) { _data[idx] = static_cast(idx); } + ++_generation; } void clear() { _owned.clear(); _data = nullptr; _size = 0; + ++_generation; } size_t size() const { return _size; } @@ -98,7 +102,29 @@ class SelectionVector { return _data[idx]; } - void set_index(size_t idx, Index value) { _data[idx] = value; } + void set_index(size_t idx, Index value) { + _data[idx] = value; + ++_generation; + } + + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { + DORIS_CHECK(filter != nullptr); + RETURN_IF_ERROR(verify(count, batch_rows)); + if (_filter_generation != _generation || _filter_count != count || + _filter_batch_rows != batch_rows) { + // Selection is shared by all readers in one scheduler batch. Cache its dense bitmap so + // a wide lazy projection does not rebuild the same O(batch_rows) filter per column. + _filter.assign(static_cast(batch_rows), 0); + for (size_t idx = 0; idx < count; ++idx) { + _filter[get_index(idx)] = 1; + } + _filter_generation = _generation; + _filter_count = count; + _filter_batch_rows = batch_rows; + } + *filter = _filter.data(); + return Status::OK(); + } Status verify(size_t count, int64_t batch_rows) const { if (batch_rows < 0) { @@ -135,13 +161,19 @@ class SelectionVector { std::vector _owned; Index* _data = nullptr; size_t _size = 0; + uint64_t _generation = 0; + mutable std::vector _filter; + mutable uint64_t _filter_generation = std::numeric_limits::max(); + mutable size_t _filter_count = 0; + mutable int64_t _filter_batch_rows = -1; }; -inline std::vector selection_to_ranges(const SelectionVector& selection, - uint16_t selected_rows) { - std::vector ranges; +inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, + std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); if (selected_rows == 0) { - return ranges; + return; } int64_t range_start = selection.get_index(0); @@ -152,11 +184,17 @@ inline std::vector selection_to_ranges(const SelectionVector& selectio previous = current; continue; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); range_start = current; previous = current; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); +} + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + selection_to_ranges(selection, selected_rows, &ranges); return ranges; } diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp index 783ccbf95034fb..619e9959a1bfa4 100644 --- a/be/src/format_v2/table/hive_reader.cpp +++ b/be/src/format_v2/table/hive_reader.cpp @@ -60,7 +60,7 @@ bool is_file_column_position_slot(const TFileScanSlotInfo& slot_info, if (slot_info.__isset.is_file_slot) { return slot_info.is_file_slot; } - return true; + return !slot_info.__isset.category || slot_info.category != TColumnCategory::PARTITION_KEY; } bool is_hive1_orc_column_name(std::string_view name) { @@ -102,11 +102,16 @@ void add_name_mapping(std::vector* name_mapping, const std::string& } // namespace Status HiveReader::prepare_split(const format::SplitReadOptions& options) { - if (options.current_split_format != _format) { - return Status::InternalError( - "Hive scan expects all splits to use the same file format, " - "initialized_format={}, current_split_format={}", - static_cast(_format), static_cast(options.current_split_format)); + { + // Keep derived validation visible without overlapping the base scopes on the same timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } } return format::TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 6c8917d867074c..d05cfe4b03e357 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -28,13 +28,20 @@ namespace doris::format::hudi { Status HudiReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.hudi_params && - options.current_range.table_format_params.hudi_params.__isset.schema_id) { - _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -66,6 +73,8 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { + // A newly selected child initializes against the same scanner profile. Keep hybrid dispatch + // outside those shared counters so first-split initialization is counted exactly once. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -120,7 +129,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -128,7 +145,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt format::FileFormat file_format; RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::hudi::HudiReader::create_unique(); + } +#else _native_reader = format::hudi::HudiReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index 77a71cd79ba166..d81e8c80e93ed3 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -73,6 +74,12 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -88,6 +95,10 @@ class HudiHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle native parquet/orc splits std::unique_ptr _jni_reader; // handle MOR JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::hudi diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index 2f808188f0d7c2..b422243707e7db 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -162,12 +162,17 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + // The inner delete-file reader has distinct counters, so the outer preparation can safely + // contain its cache miss/open work without re-entering the same RuntimeProfile timer. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); _current_range = options.current_range; _has_split = true; return _init_split(); } Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); @@ -308,6 +313,15 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { std::vector projected_columns; RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + static constexpr const char* kPositionReaderProfile = "IcebergPositionDeleteFileReader"; + if (_position_reader_profile == nullptr) { + _position_reader_profile = _scanner_profile->get_child(kPositionReaderProfile); + if (_position_reader_profile == nullptr) { + // The outer system-table reader calls the inner reader synchronously. Giving both the + // same profile would nest identical counter pointers and double-count every timer. + _position_reader_profile = _scanner_profile->create_child(kPositionReaderProfile); + } + } _position_reader = std::make_unique(); RETURN_IF_ERROR(_position_reader->init({ .projected_columns = std::move(projected_columns), @@ -316,7 +330,7 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, - .scanner_profile = _scanner_profile, + .scanner_profile = _position_reader_profile, .file_slot_descs = nullptr, .push_down_agg_type = TPushAggOp::type::NONE, .condition_cache_digest = 0, diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h index 9c802e726053cb..09756daaabe4da 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -76,6 +76,7 @@ class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; std::unique_ptr _position_reader; + RuntimeProfile* _position_reader_profile = nullptr; std::vector _read_columns; ColumnPtr _partition_value; roaring::Roaring64Map _dv_positions; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index d83a8c63dc8547..407d5924e9f64a 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -36,7 +36,7 @@ #include "core/field.h" #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" -#include "format_v2/deletion_vector_reader.h" +#include "format/table/deletion_vector_reader.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/orc/orc_reader.h" @@ -44,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/debug_points.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -195,7 +196,7 @@ static std::string iceberg_params_debug_string(const std::optionalkey = ::doris::format::build_iceberg_deletion_vector_cache_key(data_file_path, - *deletion_vector); + desc->key = build_iceberg_deletion_vector_cache_key(data_file_path, *deletion_vector); desc->path = deletion_vector->path; desc->start_offset = deletion_vector->content_offset; desc->size = static_cast(bytes_read); @@ -650,10 +658,13 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && scan_params.enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = + scan_params.__isset.enable_mapping_varbinary && scan_params.enable_mapping_varbinary; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + // Delete and data files must parse raw binary fields with the same scan-level mapping. *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, enable_mapping_varbinary); } else { *reader = std::make_unique(system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index dde136e0de5763..cf864188074dcd 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -33,15 +33,13 @@ namespace doris { class Block; +struct DeleteFileDesc; namespace io { struct FileDescription; struct FileSystemProperties; } // namespace io } // namespace doris -namespace doris::format { -struct DeleteFileDesc; -} namespace doris::format::iceberg { // Iceberg table-level reader. diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index b22441c2b696e9..942fbb234269fc 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -23,9 +23,9 @@ #include #include "exprs/vexpr_context.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/jni/paimon_jni_reader.h" #include "format_v2/table/schema_history_util.h" #include "gen_cpp/PlanNodes_types.h" @@ -33,12 +33,19 @@ namespace doris::format::paimon { Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -78,7 +85,7 @@ Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_d size_t bytes_read = 0; RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); - desc->key = ::doris::format::build_paimon_deletion_vector_cache_key(deletion_file); + desc->key = build_paimon_deletion_vector_cache_key(deletion_file); desc->path = deletion_file.path; desc->start_offset = deletion_file.offset; desc->size = static_cast(bytes_read); @@ -93,6 +100,8 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { + // Child initialization uses the scanner profile too; hybrid dispatch must not nest the same + // timer around the first native or JNI child and double-count that initialization. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -147,7 +156,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -158,7 +175,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO DCHECK(file_format == format::FileFormat::PARQUET || file_format == format::FileFormat::ORC); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::paimon::PaimonReader::create_unique(); + } +#else _native_reader = format::paimon::PaimonReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 66f96dc7898c11..ad29075fed1f08 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -17,11 +17,12 @@ #pragma once +#include #include #include "format_v2/table_reader.h" -namespace doris::format { +namespace doris { struct DeleteFileDesc; } namespace doris::format::paimon { @@ -84,6 +85,12 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -96,6 +103,10 @@ class PaimonHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle parquet/orc native splits std::unique_ptr _jni_reader; // handle serialized JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::paimon diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index c67cece6e05b8c..db278d9b0cedda 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -37,6 +37,7 @@ #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/timezone_utils.h" @@ -177,7 +178,27 @@ RemoteDorisFileReader::~RemoteDorisFileReader() { static_cast(close()); } +void RemoteDorisFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* remote_profile = "RemoteDorisFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, remote_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisOpenStreamTime", remote_profile, 1); + _next_batch_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisNextBatchTime", remote_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisMaterializeTime", remote_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisFilterTime", remote_profile, 1); +} + Status RemoteDorisFileReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); (void)state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); @@ -186,6 +207,7 @@ Status RemoteDorisFileReader::init(RuntimeState* state) { } Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_schema != nullptr); file_schema->clear(); file_schema->reserve(_file_slot_descs.size()); @@ -206,6 +228,8 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch } Status RemoteDorisFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); RETURN_IF_ERROR(_open_stream()); _eof = false; @@ -213,6 +237,7 @@ Status RemoteDorisFileReader::open(std::shared_ptr request) { } Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -223,21 +248,32 @@ Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* e *rows = 0; *eof = false; std::shared_ptr batch; - RETURN_IF_ERROR(_stream->next(&batch)); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } if (batch == nullptr) { *eof = true; _eof = true; return Status::OK(); } - RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR( - apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } return Status::OK(); } Status RemoteDorisFileReader::close() { + SCOPED_TIMER(_total_time); if (_stream != nullptr) { RETURN_IF_ERROR(_stream->close()); _stream.reset(); @@ -349,7 +385,12 @@ Status RemoteDorisReader::init(TableReadOptions&& options) { } Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { - RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + } return TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index b4dd2a505a95ad..79c37a29be4a06 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -71,6 +71,7 @@ class RemoteDorisFileReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _open_stream(); Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, size_t* rows) const; @@ -83,6 +84,12 @@ class RemoteDorisFileReader final : public FileReader { const std::vector _file_slot_descs; RemoteDorisStreamFactory _stream_factory; cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 8ae0a70242963c..4beaf8c9ff5550 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -39,17 +39,18 @@ #include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/iceberg_scan_semantics.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/delimited_text/csv_reader.h" #include "format_v2/delimited_text/text_reader.h" #include "format_v2/json/json_reader.h" #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" #include "util/string_util.h" @@ -642,34 +643,15 @@ std::optional TableReader::_find_current_table_column_by_field } Status TableReader::init(TableReadOptions&& options) { - _scan_params = options.scan_params; - _format = options.format; - _io_ctx = options.io_ctx; - _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; - _file_slot_descs = options.file_slot_descs; - _push_down_agg_type = options.push_down_agg_type; - _push_down_count_columns = options.push_down_count_columns; - _initial_condition_cache_digest = options.condition_cache_digest; - _condition_cache_digest = _initial_condition_cache_digest; - _projected_columns = std::move(options.projected_columns); - if (supports_iceberg_scan_semantics_v1(_scan_params)) { - for (auto& projected_column : _projected_columns) { - const auto* schema_field = find_external_root_field(_scan_params, projected_column); - if (schema_field != nullptr) { - attach_full_schema_identity( - &projected_column, - build_schema_identity_from_external_field(*schema_field)); - } - } - } - _system_properties = create_system_properties(_scan_params); - _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); - if (_scanner_profile != nullptr) { - static const char* table_profile = "TableReader"; - ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); + static const char* table_profile = file_scan_profile::TABLE_READER; + static const char* file_reader_profile = file_scan_profile::FILE_READER; + _profile.total_timer = hierarchy.table_reader; + _profile.file_reader_total_timer = hierarchy.file_reader; + _profile.init_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "InitTime", table_profile, 1); _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", TUnit::UNIT, table_profile, 1); _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", @@ -702,11 +684,57 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); _profile.open_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); - _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( - _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); - _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( - _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); + _profile.runtime_filter_partition_prune_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", table_profile, 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, + table_profile, 1); + _profile.close_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CloseTime", table_profile, 1); + // Lifecycle timer names remain globally unique because RuntimeProfile's visual hierarchy + // does not namespace counters that share the same display parent. + _profile.file_reader_init_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderInitTime", file_reader_profile, 1); + _profile.file_reader_schema_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetSchemaTime", file_reader_profile, 1); + _profile.file_reader_mapper_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCreateColumnMapperTime", file_reader_profile, 1); + _profile.file_reader_open_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderOpenTime", file_reader_profile, 1); + _profile.file_reader_get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetBlockTime", file_reader_profile, 1); + _profile.file_reader_aggregate_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderAggregatePushDownTime", file_reader_profile, 1); + _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); + } + // Establish lifecycle timers before consuming options or constructing filesystem properties; + // placing these scopes at the tail records only scope teardown and hides expensive init work. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; + _initial_condition_cache_digest = options.condition_cache_digest; + _condition_cache_digest = _initial_condition_cache_digest; + _projected_columns = std::move(options.projected_columns); + if (supports_iceberg_scan_semantics_v1(_scan_params)) { + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity( + &projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } } + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + _conjuncts = std::move(options.conjuncts); return Status::OK(); } @@ -720,16 +748,17 @@ Status TableReader::_build_table_filters_from_conjuncts() { // `_table_filters` omits expressions without slot references, but such an expression still // occupies a position in the row-level conjunct order. Record how many localized filters // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. + // slotless non-deterministic/error-preserving barrier. Unsafe predicates remain solely on + // Scanner's original row-level path because localizing a clone would execute their state + // twice with independent state. if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { in_safe_prefix = false; } - if (!in_safe_prefix) { - continue; - } RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); - _constant_pruning_safe_filter_count = _table_filters.size(); + if (in_safe_prefix) { + _constant_pruning_safe_filter_count = _table_filters.size(); + } } return Status::OK(); } @@ -855,7 +884,12 @@ Status TableReader::create_next_reader(bool* eos) { if (_batch_size > 0) { _data_reader.reader->set_batch_size(_batch_size); } - Status st = _data_reader.reader->init(_runtime_state); + Status st; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + st = _data_reader.reader->init(_runtime_state); + } if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { *eos = true; @@ -886,10 +920,16 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; if (_format == FileFormat::PARQUET) { + // V2 must honor the scan contract directly; otherwise Hive STRING columns backed by an + // unannotated BYTE_ARRAY are silently exposed as VARBINARY and predicate bytes no longer + // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); return Status::OK(); } if (_format == FileFormat::ORC) { @@ -960,6 +1000,7 @@ std::unique_ptr create_file_description(const TFileRangeDes } Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; @@ -994,6 +1035,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _deletion_vector = nullptr; _aggregate_pushdown_tried = false; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; _current_reader_reached_eof = false; RETURN_IF_ERROR(_evaluate_partition_prune_conjuncts(options.partition_prune_conjuncts, diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index c26ba086788020..7adbb3b0a5225c 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -54,9 +54,9 @@ #include "exprs/vexpr_context.h" #include "exprs/vexpr_fwd.h" #include "exprs/vslot_ref.h" +#include "format/table/deletion_vector.h" #include "format_v2/column_data.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" @@ -69,11 +69,11 @@ namespace doris { class Block; +struct DeleteFileDesc; class RuntimeState; } // namespace doris namespace doris::format { -struct DeleteFileDesc; using DeleteRows = std::vector; @@ -99,6 +99,8 @@ struct ProjectedColumnBuildContext { }; struct ReadProfile { + RuntimeProfile::Counter* total_timer = nullptr; + RuntimeProfile::Counter* init_timer = nullptr; RuntimeProfile::Counter* num_delete_files = nullptr; RuntimeProfile::Counter* num_delete_rows = nullptr; RuntimeProfile::Counter* parse_delete_file_time = nullptr; @@ -115,6 +117,15 @@ struct ReadProfile { RuntimeProfile::Counter* open_reader_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; + RuntimeProfile::Counter* close_timer = nullptr; + RuntimeProfile::Counter* file_reader_total_timer = nullptr; + RuntimeProfile::Counter* file_reader_init_timer = nullptr; + RuntimeProfile::Counter* file_reader_schema_timer = nullptr; + RuntimeProfile::Counter* file_reader_mapper_timer = nullptr; + RuntimeProfile::Counter* file_reader_open_timer = nullptr; + RuntimeProfile::Counter* file_reader_get_block_timer = nullptr; + RuntimeProfile::Counter* file_reader_aggregate_timer = nullptr; + RuntimeProfile::Counter* file_reader_close_timer = nullptr; }; struct TableReadOptions { @@ -219,6 +230,10 @@ class TableReader { // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. virtual Status abort_split() { + // Ignored open failures still spend time closing partially initialized readers. Include + // that recovery path in the common lifecycle profile so NOT_FOUND cannot become invisible. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader != nullptr) { RETURN_IF_ERROR(close_current_reader()); } else { @@ -227,6 +242,7 @@ class TableReader { } _delete_rows = nullptr; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; _current_split_pruned = false; return Status::OK(); @@ -236,6 +252,7 @@ class TableReader { // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for // table-format-specific behavior. virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -253,6 +270,10 @@ class TableReader { RETURN_IF_ERROR(_read_table_level_count(block, eos)); return Status::OK(); } + if (_is_file_level_count_active()) { + RETURN_IF_ERROR(_read_file_level_count(block, eos)); + return Status::OK(); + } RETURN_IF_ERROR(create_next_reader(eos)); if (!_data_reader.reader) { DCHECK(*eos); @@ -285,8 +306,12 @@ class TableReader { _data_reader.block_template.clear_column_data( cast_set(_data_reader.file_block_layout.size())); size_t current_rows = 0; - RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, - ¤t_rows, ¤t_eof)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_get_block_timer); + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + } const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; if (current_rows == 0) { if (current_eof) { @@ -317,12 +342,15 @@ class TableReader { // Close the table reader and the currently active file reader. Subclasses that hold additional // table-format resources should override this and call TableReader::close() first. virtual Status close() { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader) { RETURN_IF_ERROR(close_current_reader()); } _current_task.reset(); _current_file_description.reset(); _remaining_table_level_count = -1; + _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; return Status::OK(); } @@ -380,7 +408,11 @@ class TableReader { SCOPED_TIMER(_profile.open_reader_timer); // 1. Get file schema and create column mapping. std::vector file_schema; - RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_schema_timer); + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + } // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the // file schema before column mapping when the table format maps columns by field id. RETURN_IF_ERROR(annotate_file_schema(&file_schema)); @@ -388,7 +420,11 @@ class TableReader { _mapper_options.mode = mapping_mode(); configure_mapper_options(&_mapper_options); - _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_mapper_timer); + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + } DORIS_CHECK(_data_reader.column_mapper != nullptr); RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, _partition_values, file_schema)); @@ -474,7 +510,11 @@ class TableReader { VLOG_DEBUG << "TableReader debug: " << debug_string(); } RETURN_IF_ERROR(_open_mapping_exprs()); - RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_open_timer); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + } RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); return Status::OK(); } @@ -597,39 +637,64 @@ class TableReader { bool _is_table_level_count_active() const { return _remaining_table_level_count >= 0; } + bool _is_file_level_count_active() const { return _remaining_file_level_count >= 0; } + Status _materialize_count_rows(size_t rows, Block* block) const { DORIS_CHECK(block != nullptr); DORIS_CHECK(block->columns() > 0 || rows == 0); for (size_t column_idx = 0; column_idx < block->columns(); ++column_idx) { auto column = block->get_by_position(column_idx).type->create_column(); - column->resize(rows); + if (auto* nullable = check_and_get_column(*column)) { + // Metadata COUNT emits synthetic input rows for the unchanged upper aggregate. + // They must be non-NULL for COUNT(nullable_col), and constructing them explicitly + // also keeps every nullable null map boolean-valid in debug/ASAN block checks. + nullable->get_nested_column().insert_many_defaults(rows); + nullable->get_null_map_data().resize_fill(rows, 0); + } else { + column->insert_many_defaults(rows); + } block->replace_by_position(column_idx, std::move(column)); } return Status::OK(); } - Status _read_table_level_count(Block* block, bool* eos) { + Status _materialize_next_count_batch(int64_t* remaining_rows, Block* block) const { + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows > 0); + const int64_t batch_size = _runtime_state == nullptr + ? *remaining_rows + : static_cast(_runtime_state->batch_size()); + const auto rows = std::min(*remaining_rows, batch_size); + RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); + *remaining_rows -= rows; + return Status::OK(); + } + + Status _read_count_batch(int64_t* remaining_rows, Block* block, bool* eos) { DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(_push_down_agg_type == TPushAggOp::type::COUNT); - DORIS_CHECK(_remaining_table_level_count >= 0); - if (_remaining_table_level_count == 0) { - _remaining_table_level_count = -1; + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows >= 0); + if (*remaining_rows == 0) { + *remaining_rows = -1; _current_task.reset(); *eos = true; return Status::OK(); } - - const int64_t batch_size = _runtime_state == nullptr - ? _remaining_table_level_count - : static_cast(_runtime_state->batch_size()); - const auto rows = std::min(_remaining_table_level_count, batch_size); - RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); - _remaining_table_level_count -= rows; + RETURN_IF_ERROR(_materialize_next_count_batch(remaining_rows, block)); *eos = false; return Status::OK(); } + Status _read_table_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_table_level_count, block, eos); + } + + Status _read_file_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_file_level_count, block, eos); + } + void _append_file_scan_column(FileScanRequest* request, LocalColumnId column_id, std::vector* scan_columns) { DORIS_CHECK(request != nullptr); @@ -681,7 +746,11 @@ class TableReader { // close(), so it should remain idempotent. virtual Status close_current_reader() { _finalize_reader_condition_cache(); - RETURN_IF_ERROR(_data_reader.reader->close()); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + RETURN_IF_ERROR(_data_reader.reader->close()); + } _data_reader.reader.reset(); if (_data_reader.column_mapper != nullptr) { _data_reader.column_mapper->clear(); @@ -708,15 +777,12 @@ class TableReader { Status finalize_chunk(Block* block, const size_t rows) { SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; - for (const auto& mapping : _data_reader.column_mapper->mappings()) { + const auto& mappings = _data_reader.column_mapper->mappings(); + for (const auto& mapping : mappings) { ColumnPtr column; RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column)); - // Keep projection columns shared with the file block. Before the next read, - // clear_column_data() replaces shared columns with clone_empty(), so forcing a mutable - // deep copy here only duplicates the just-read batch (which can be several GB for - // nested string/map columns). - block->replace_by_position(idx, std::move(column)); + &column, idx + 1 == mappings.size())); + block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } RETURN_IF_ERROR(materialize_virtual_columns(block)); @@ -966,15 +1032,29 @@ class TableReader { FileAggregateRequest file_request; RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); FileAggregateResult file_result; - const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_aggregate_timer); + status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + } if (status.is()) { return Status::OK(); } RETURN_IF_ERROR(status); - RETURN_IF_ERROR( - _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); if (_push_down_agg_type == TPushAggOp::type::COUNT) { + DORIS_CHECK(file_result.count >= 0); + // The upper aggregate consumes synthetic input rows, but emitting the whole metadata + // count in one block bypasses the runtime batch contract and can allocate by file size. + // Keep the remaining cardinality as split state and expose at most one batch per call. + _remaining_file_level_count = file_result.count; _current_split_uses_metadata_count = true; + if (_remaining_file_level_count > 0) { + RETURN_IF_ERROR(_materialize_next_count_batch(&_remaining_file_level_count, block)); + } + } else { + RETURN_IF_ERROR( + _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); } *pushed_down = true; RETURN_IF_ERROR(close_current_reader()); @@ -1054,6 +1134,22 @@ class TableReader { return true; } + static ColumnPtr _detach_column(ColumnPtr column) { + DORIS_CHECK(column.get() != nullptr); + return IColumn::mutate(std::move(column)); + } + + static ColumnPtr _take_and_detach_block_column(Block* block, int position) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(position >= 0 && position < static_cast(block->columns())); + auto& source = block->get_by_position(position); + ColumnPtr column = source.column; + // The final mapping no longer needs the file block. Release its COW owner before mutate(), + // otherwise nested MAP/STRING columns are deep-copied and a multi-GB payload can OOM. + block->replace_by_position(position, source.type->create_column()); + return _detach_column(std::move(column)); + } + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); @@ -1197,7 +1293,8 @@ class TableReader { } Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, - const size_t rows, ColumnPtr* column) { + const size_t rows, ColumnPtr* column, + bool take_projection_result = false) { if (!mapping.is_trivial && mapping.file_local_id.has_value() && !mapping.child_mappings.empty()) { DCHECK(mapping.projection != nullptr); @@ -1210,7 +1307,9 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; + ColumnPtr result_column = take_projection_result + ? _take_and_detach_block_column(current_block, res_id) + : current_block->get_by_position(res_id).column; RETURN_IF_ERROR( _materialize_complex_mapping_column(mapping, result_column, rows, column)); return Status::OK(); @@ -1229,8 +1328,12 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; - *column = std::move(result_column); + if (take_projection_result) { + *column = _take_and_detach_block_column(current_block, res_id); + } else { + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + } return Status::OK(); } if (mapping.default_expr != nullptr) { @@ -1240,7 +1343,7 @@ class TableReader { mapping.default_expr, current_block, &result)); ColumnPtr result_column = result.column; RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); } else { DORIS_CHECK(mapping.constant_index.has_value()); Block eval_block; @@ -1251,12 +1354,12 @@ class TableReader { mapping.default_expr, &eval_block, &result)); ColumnPtr result_column = result.column; RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); } return Status::OK(); } ColumnPtr result_column = mapping.table_type->create_column_const_with_default_value(rows); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); return Status::OK(); } @@ -1277,7 +1380,7 @@ class TableReader { RETURN_IF_ERROR(_materialize_map_mapping_column(mapping, file_column, rows, column)); break; default: - *column = file_column; + *column = _detach_column(file_column); break; } return Status::OK(); @@ -1560,13 +1663,7 @@ class TableReader { Status _materialize_aggregate_pushdown_rows(TPushAggOp::type agg_type, const FileAggregateResult& file_result, Block* block) { - if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown is not a final count value. It emits `count` default rows so the - // upper COUNT(*) aggregate can count them and produce the final result, including - // zero rows when count is 0. - DORIS_CHECK(file_result.count >= 0); - return _materialize_count_rows(cast_set(file_result.count), block); - } + DORIS_CHECK(agg_type == TPushAggOp::type::MINMAX); // MIN/MAX pushdown emits two rows, min first and max second, for each projected column. // The upper MIN/MAX aggregate consumes those two rows to produce the final aggregate value. DORIS_CHECK(file_result.columns.size() == _data_reader.column_mapper->mappings().size()); @@ -1606,9 +1703,10 @@ class TableReader { for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); ++column_idx) { ColumnPtr table_column; - RETURN_IF_ERROR( - _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], - &file_block, 2, &table_column)); + RETURN_IF_ERROR(_materialize_mapping_column( + _data_reader.column_mapper->mappings()[column_idx], &file_block, 2, + &table_column, + column_idx + 1 == _data_reader.column_mapper->mappings().size())); block->replace_by_position(column_idx, std::move(table_column)); } return Status::OK(); @@ -1675,6 +1773,7 @@ class TableReader { int64_t _condition_cache_hit_count = 0; bool _current_reader_reached_eof = false; int64_t _remaining_table_level_count = -1; + int64_t _remaining_file_level_count = -1; // True only after the active split selects a table-level row-count shortcut or successfully // materializes COUNT rows from file metadata. FileScannerV2 uses this result, rather than the // raw aggregate opcode, to keep adaptive batching enabled for normal row-scan fallbacks. diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 30e1336e2a8bd7..94eab329687f08 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -124,9 +124,11 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren return diff; } -FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) { +FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, + const std::string& parent_counter) + : _profile(profile) { static const char* cache_profile = "FileCache"; - ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); + total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, cache_profile, parent_counter.c_str(), 2); num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, @@ -200,6 +202,12 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) { } void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { + // These are the outer cache-path phases. Their sum keeps the group timer actionable instead of + // displaying zero while individual cache IO and coordination timers are non-zero. + COUNTER_UPDATE(total_time, statistics->local_io_timer + statistics->remote_io_timer + + statistics->peer_io_timer + statistics->remote_wait_timer + + statistics->write_cache_io_timer + + statistics->cache_get_or_set_timer); COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); COUNTER_UPDATE(num_peer_io_total, statistics->num_peer_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 297b2532c026b8..7015f4b04fb64f 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -70,6 +70,8 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren const FileCacheStatistics& previous); struct FileCacheProfileReporter { + RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; RuntimeProfile::Counter* num_peer_io_total = nullptr; @@ -110,7 +112,9 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* segment_footer_index_remote_io_timer = nullptr; RuntimeProfile::Counter* segment_footer_index_peer_io_timer = nullptr; - FileCacheProfileReporter(RuntimeProfile* profile); + explicit FileCacheProfileReporter( + RuntimeProfile* profile, + const std::string& parent_counter = RuntimeProfile::ROOT_COUNTER); void update(const FileCacheStatistics* statistics) const; }; diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index e8005ef821ba3a..91eafdba4319fe 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -32,6 +32,7 @@ #include "common/status.h" #include "core/custom_allocator.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -657,7 +658,9 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File std::function sync_buffer = nullptr; if (profile != nullptr) { const char* prefetch_buffered_reader = "PrefetchBufferedReader"; - ADD_TIMER(profile, prefetch_buffered_reader); + auto* total_time = + ADD_CHILD_TIMER(profile, prefetch_buffered_reader, + file_scan_profile::parent_or_root(profile, file_scan_profile::IO)); auto copy_time = ADD_CHILD_TIMER(profile, "CopyTime", prefetch_buffered_reader); auto read_time = ADD_CHILD_TIMER(profile, "ReadTime", prefetch_buffered_reader); auto prefetch_request_io = @@ -669,6 +672,7 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File auto request_bytes = ADD_CHILD_COUNTER(profile, "RequestBytes", TUnit::BYTES, prefetch_buffered_reader); sync_buffer = [=](PrefetchBuffer& buf) { + COUNTER_UPDATE(total_time, buf._statis.copy_time + buf._statis.read_time); COUNTER_UPDATE(copy_time, buf._statis.copy_time); COUNTER_UPDATE(read_time, buf._statis.read_time); COUNTER_UPDATE(prefetch_request_io, buf._statis.prefetch_request_io); @@ -926,7 +930,9 @@ RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileRead if (_profile != nullptr) { const char* random_profile = "RangeCacheFileReader"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, @@ -987,6 +993,7 @@ Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* b void RangeCacheFileReader::_collect_profile_before_close() { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _cache_statistics.request_time); COUNTER_UPDATE(_request_io, _cache_statistics.request_io); COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); COUNTER_UPDATE(_request_time, _cache_statistics.request_time); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index f88fb2a575fce6..04566bea0951da 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -38,6 +38,7 @@ #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" #include "io/io_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -181,6 +182,7 @@ class RangeCacheFileReader : public io::FileReader { bool _closed = false; RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _request_time = nullptr; RuntimeProfile::Counter* _read_to_cache_time = nullptr; @@ -303,7 +305,9 @@ class MergeRangeFileReader : public io::FileReader { if (_profile != nullptr) { const char* random_profile = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -352,6 +356,7 @@ class MergeRangeFileReader : public io::FileReader { void _collect_profile_before_close() override { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -365,6 +370,7 @@ class MergeRangeFileReader : public io::FileReader { } private: + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _copy_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 7fd8b5df36e6f5..505a2da1976343 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -32,6 +32,7 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/hdfs_util.h" +#include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" @@ -84,7 +85,9 @@ HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache:: if (_profile != nullptr && is_hdfs(_fs_name)) { #ifdef USE_HADOOP_HDFS const char* hdfs_profile_name = "HdfsIO"; - ADD_TIMER(_profile, hdfs_profile_name); + _total_read_time = + ADD_CHILD_TIMER(_profile, hdfs_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); _hdfs_profile.total_bytes_read = ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES, hdfs_profile_name); _hdfs_profile.total_local_bytes_read = @@ -118,6 +121,7 @@ Status HdfsFileReader::close() { Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) { + SCOPED_TIMER(_total_read_time); auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); if (!st.ok()) { _handle = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 08f98bca29af0c..7bb73f30909452 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -88,6 +88,7 @@ class HdfsFileReader final : public FileReader { CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle std::atomic _closed = false; RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* _total_read_time = nullptr; int64_t _mtime; #ifdef USE_HADOOP_HDFS HDFSProfile _hdfs_profile; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..af8dde36d2df50 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -37,6 +37,7 @@ #include "io/fs/err_utils.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -214,7 +215,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea void S3FileReader::_collect_profile_before_close() { if (_profile != nullptr) { const char* s3_profile_name = "S3Profile"; - ADD_TIMER(_profile, s3_profile_name); + auto* total_time = + ADD_CHILD_TIMER(_profile, s3_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); RuntimeProfile::Counter* total_get_request_counter = ADD_CHILD_COUNTER(_profile, "TotalGetRequest", TUnit::UNIT, s3_profile_name); RuntimeProfile::Counter* too_many_request_err_counter = @@ -231,6 +234,7 @@ void S3FileReader::_collect_profile_before_close() { COUNTER_UPDATE(too_many_request_sleep_time, _s3_stats.too_many_request_sleep_time_ms); COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read); COUNTER_UPDATE(total_get_request_time_ns, _s3_stats.total_get_request_time_ns); + COUNTER_UPDATE(total_time, _s3_stats.total_get_request_time_ns); } } diff --git a/be/src/runtime/file_scan_profile.h b/be/src/runtime/file_scan_profile.h new file mode 100644 index 00000000000000..4ec71c4f4820d4 --- /dev/null +++ b/be/src/runtime/file_scan_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::file_scan_profile { + +inline constexpr const char* SCANNER = "FileScannerV2"; +inline constexpr const char* TABLE_READER = "TableReader"; +inline constexpr const char* FILE_READER = "FileReader"; +inline constexpr const char* IO = "IO"; + +struct Hierarchy { + RuntimeProfile::Counter* scanner = nullptr; + RuntimeProfile::Counter* table_reader = nullptr; + RuntimeProfile::Counter* file_reader = nullptr; + RuntimeProfile::Counter* io = nullptr; +}; + +inline Hierarchy ensure_hierarchy(RuntimeProfile* profile) { + if (profile == nullptr) { + return {}; + } + // RuntimeProfile stores one flat counter namespace and a separate parent map. Create every + // layer in order so later format and IO profiles cannot accidentally become scanner siblings. + auto* scanner = ADD_TIMER_WITH_LEVEL(profile, SCANNER, 1); + auto* table_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, TABLE_READER, SCANNER, 1); + auto* file_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, FILE_READER, TABLE_READER, 1); + auto* io = ADD_CHILD_TIMER_WITH_LEVEL(profile, IO, FILE_READER, 1); + return {scanner, table_reader, file_reader, io}; +} + +inline const char* parent_or_root(RuntimeProfile* profile, const char* preferred_parent) { + return profile != nullptr && profile->get_counter(preferred_parent) != nullptr + ? preferred_parent + : RuntimeProfile::ROOT_COUNTER.c_str(); +} + +} // namespace doris::file_scan_profile diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 75f05269853039..a12c8743ee4cfe 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -155,6 +155,18 @@ RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id, DCHECK(_query_mem_tracker != nullptr); } +RuntimeState::RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) + : _profile(""), + _load_channel_profile(""), + _obj_pool(new ObjectPool()), + _unreported_error_idx(0), + _per_fragment_instance_idx(0) { + Status status = init(TUniqueId(), query_options, query_globals, nullptr); + _exec_env = ExecEnv::GetInstance(); + init_mem_trackers(""); + DCHECK(status.ok()); +} + RuntimeState::RuntimeState(const TQueryGlobals& query_globals) : _profile(""), _load_channel_profile(""), diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 4fdce6c07c2d0d..40ba041ffab388 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -100,20 +100,16 @@ class RuntimeState { const TQueryGlobals& query_globals, ExecEnv* exec_env, const std::shared_ptr& query_mem_tracker); + // Keep this production-visible because master format_v2 constructs a lightweight state for + // page-cache reads; a BE_TEST-only overload would hide release-build incompatibilities. + RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals); + // RuntimeState for executing expr in fe-support. RuntimeState(const TQueryGlobals& query_globals); // for job task only RuntimeState(); -#ifdef BE_TEST - // Compatibility constructor for format_v2 tests backported to branch-4.1. - RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) - : RuntimeState(query_globals) { - set_query_options(query_options); - } -#endif - // Empty d'tor to avoid issues with unique_ptr. MOCK_DEFINE(virtual) ~RuntimeState(); diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index f800f72ebbbdcb..5229527d1122f4 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -770,11 +770,19 @@ class SnappyBlockCompression : public BlockCompressionCodec { } Status decompress(const Slice& input, Slice* output) override { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(input.data, input.size, &uncompressed_size)) { + return Status::InvalidArgument("Fail to get Snappy uncompressed length"); + } + // RawUncompress has no capacity argument, so reject an undersized destination first. + if (uncompressed_size > output->size) { + return Status::InvalidArgument("Snappy output size {} exceeds buffer capacity {}", + uncompressed_size, output->size); + } if (!snappy::RawUncompress(input.data, input.size, output->data)) { return Status::InvalidArgument("Fail to do Snappy decompress"); } - // NOTE: GetUncompressedLength only takes O(1) time - snappy::GetUncompressedLength(input.data, input.size, &output->size); + output->size = uncompressed_size; return Status::OK(); } diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 2987d1dc099ce8..3c1af89e228f3f 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -376,6 +376,17 @@ void CpuInfo::_get_cache_info(long cache_sizes[NUM_CACHE_LEVELS], #endif } +long CpuInfo::get_l2_cache_size() { + static const long l2_cache_size = [] { + long cache_sizes[NUM_CACHE_LEVELS] = {}; + long cache_line_sizes[NUM_CACHE_LEVELS] = {}; + _get_cache_info(cache_sizes, cache_line_sizes); + constexpr long DEFAULT_L2_CACHE_SIZE = 256 * 1024; + return cache_sizes[L2_CACHE] > 0 ? cache_sizes[L2_CACHE] : DEFAULT_L2_CACHE_SIZE; + }(); + return l2_cache_size; +} + std::string CpuInfo::debug_string() { DCHECK(initialized_); std::stringstream stream; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index 19548ecc964659..c75b814721cc57 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -146,6 +146,10 @@ class CpuInfo { return model_name_; } + // Some virtualized hosts do not expose cache topology. Always return a positive value so hot + // decode paths can use this threshold without repeating platform-specific guards. + static long get_l2_cache_size(); + static std::string debug_string(); /// A utility class for temporarily disabling CPU features. Usage: diff --git a/be/src/util/jdbc_utils.cpp b/be/src/util/jdbc_utils.cpp new file mode 100644 index 00000000000000..cf091f6c9ff425 --- /dev/null +++ b/be/src/util/jdbc_utils.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/jdbc_utils.h" + +#include + +#include "cloud/config.h" +#include "common/config.h" +#include "runtime/plugin/cloud_plugin_downloader.h" + +namespace doris { + +Status JdbcUtils::resolve_driver_url(const std::string& url, std::string* result_url) { + // Already a full URL (e.g. "file:///path/to/driver.jar" or "hdfs://...") + if (url.find(":/") != std::string::npos) { + *result_url = url; + return Status::OK(); + } + + const char* doris_home = std::getenv("DORIS_HOME"); + if (doris_home == nullptr) { + return Status::InternalError("DORIS_HOME environment variable is not set"); + } + + std::string default_url = std::string(doris_home) + "/plugins/jdbc_drivers"; + std::string default_old_url = std::string(doris_home) + "/jdbc_drivers"; + + if (config::jdbc_drivers_dir == default_url) { + std::string target_path = default_url + "/" + url; + std::string old_target_path = default_old_url + "/" + url; + if (std::filesystem::exists(target_path)) { + *result_url = "file://" + target_path; + } else if (std::filesystem::exists(old_target_path)) { + *result_url = "file://" + old_target_path; + } else if (config::is_cloud_mode()) { + // In cloud/elastic deployments, BEs are ephemeral and driver JARs + // may not exist locally. Try downloading from cloud storage. + std::string downloaded_path; + RETURN_IF_ERROR(CloudPluginDownloader::download_from_cloud( + CloudPluginDownloader::PluginType::JDBC_DRIVERS, url, target_path, + &downloaded_path)); + *result_url = "file://" + downloaded_path; + } else { + return Status::InternalError("JDBC driver file does not exist: " + url); + } + } else { + *result_url = "file://" + config::jdbc_drivers_dir + "/" + url; + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/util/jdbc_utils.h b/be/src/util/jdbc_utils.h new file mode 100644 index 00000000000000..8c6004c4c24882 --- /dev/null +++ b/be/src/util/jdbc_utils.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" + +namespace doris { + +/** + * Utility functions for JDBC driver management. + */ +class JdbcUtils { +public: + /** + * Resolve a JDBC driver URL to an absolute file:// URL. + * + * FE sends just the JAR filename (e.g. "mysql-connector-java-8.0.25.jar"). + * This method resolves it to a full file:// URL by searching in the + * configured jdbc_drivers_dir (or the default DORIS_HOME/plugins/jdbc_drivers). + * + * If the URL already contains ":/", it is assumed to be a full URL and + * returned as-is. + * + * @param url The driver URL from FE (may be just a filename) + * @param result_url Output: the resolved file:// URL + * @return Status::OK on success, or InternalError if the file is not found + */ + static Status resolve_driver_url(const std::string& url, std::string* result_url); +}; + +} // namespace doris diff --git a/be/src/util/timezone_utils.cpp b/be/src/util/timezone_utils.cpp index 48227963cd3341..c18e3cbb7a156f 100644 --- a/be/src/util/timezone_utils.cpp +++ b/be/src/util/timezone_utils.cpp @@ -154,7 +154,14 @@ bool TimezoneUtils::find_cctz_time_zone(const std::string& timezone, cctz::time_ ctz = it->second; return true; } - return false; + // V2 readers and Iceberg writers may resolve UTC/fixed offsets before ExecEnv preloads the + // timezone cache, so retain the cache fast path but handle those self-contained zones here. + const auto normalized = to_lower_copy(timezone); + if (normalized == "utc" || normalized == "etc/utc" || normalized == "zulu") { + ctz = cctz::utc_time_zone(); + return true; + } + return parse_tz_offset_string(timezone, ctz); } bool TimezoneUtils::parse_tz_offset_string(const std::string& timezone, cctz::time_zone& ctz) { diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp index 69cf458e2fdc5f..18e4aa8a1cf848 100644 --- a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -365,7 +365,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadIntegersFromUnsignedSources) { TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalValues) { { - std::vector values = {0, 127, 255, 32767, 65535, -1}; + std::vector values = {0, 127, 255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto result = read_column(std::make_shared(), view); @@ -375,12 +375,9 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal EXPECT_EQ(0, column.get_element(0)); EXPECT_EQ(127, column.get_element(1)); EXPECT_EQ(255, column.get_element(2)); - EXPECT_EQ(255, column.get_element(3)); - EXPECT_EQ(255, column.get_element(4)); - EXPECT_EQ(255, column.get_element(5)); } { - std::vector values = {32767, 65535, -1}; + std::vector values = {32767, 65535}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 16, false); auto result = read_column(std::make_shared(), view); @@ -389,7 +386,6 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal ASSERT_EQ(values.size(), column.size()); EXPECT_EQ(32767, column.get_element(0)); EXPECT_EQ(65535, column.get_element(1)); - EXPECT_EQ(65535, column.get_element(2)); } { std::vector values = {-1}; @@ -414,17 +410,42 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal } } -TEST(DataTypeSerDeDecodedValuesTest, ReadSignedLogicalIntegersCastsPhysicalValues) { +TEST(DataTypeSerDeDecodedValuesTest, StrictSignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { std::vector values = {127, 128, 255, -1}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, true); + // Carrier bounds are metadata validation: strict scans reject malformed values, while + // permissive scans intentionally preserve the declared bit-width conversion. + view.enable_strict_mode = true; auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, StrictUnsignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { + std::vector values = {255, 256}; + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); + view.enable_strict_mode = true; + auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, + NonStrictNullableLogicalIntegerCarrierPreservesBitWidthCompatibility) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {127, 1000, -128}; + std::vector null_map(values.size(), 0); + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values, &null_map), 8, + true); + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status; - const auto& column = assert_cast(*result.column); - ASSERT_EQ(values.size(), column.size()); - EXPECT_EQ(static_cast(127), column.get_element(0)); - EXPECT_EQ(static_cast(-128), column.get_element(1)); - EXPECT_EQ(static_cast(-1), column.get_element(2)); - EXPECT_EQ(static_cast(-1), column.get_element(3)); + const auto& nullable = assert_cast(*result.column); + const auto& nested = assert_cast(nullable.get_nested_column()); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(nested.get_data(), (ColumnInt8::Container {127, -24, -128})); } TEST(DataTypeSerDeDecodedValuesTest, ReadFloatAndDouble) { @@ -1633,7 +1654,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadFieldPrimitiveValues) { TEST(DataTypeSerDeDecodedValuesTest, ReadFieldLogicalIntegerCastsPhysicalValue) { { - std::vector values = {32767}; + std::vector values = {255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto field = read_field(std::make_shared(), view); diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp new file mode 100644 index 00000000000000..b7a4329f0d261b --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -0,0 +1,891 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { +namespace { + +#pragma pack(1) +struct TestParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestParquetInt96Timestamp) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t NANOS_PER_DAY = 86400000000000LL; + +class TestParquetDecodeSource final : public ParquetDecodeSource { +public: + template + void set_fixed_values(const std::vector& values) { + _value_width = sizeof(T); + _fixed_values.resize(values.size() * sizeof(T)); + memcpy(_fixed_values.data(), values.data(), _fixed_values.size()); + } + + void set_fixed_bytes(std::vector values, size_t value_width) { + _fixed_values = std::move(values); + _value_width = value_width; + } + + void set_dictionary(std::vector values, size_t value_width, + std::vector indices) { + _dictionary = std::move(values); + _dictionary_width = value_width; + _indices = std::move(indices); + _index_offset = 0; + ++_dictionary_generation; + } + + template + void set_fixed_dictionary(const std::vector& values, std::vector indices) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + set_dictionary(std::move(encoded), sizeof(T), std::move(indices)); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); + const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; + _fixed_offset += num_values; + return consumer.consume(begin, num_values, _value_width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_LE(_binary_offset + num_values, _binary_refs.size()); + const StringRef* begin = _binary_refs.data() + _binary_offset; + _binary_offset += num_values; + return consumer.consume(begin, num_values); + } + + Status skip_values(size_t num_values) override { + _fixed_offset += num_values; + _binary_offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + ++_dictionary_decode_calls; + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + DORIS_CHECK_LE(_index_offset + num_values, _indices.size()); + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + + size_t dictionary_decode_calls() const { return _dictionary_decode_calls; } + +private: + std::vector _fixed_values; + std::vector _binary_refs; + std::vector _dictionary; + std::vector _indices; + size_t _value_width = 0; + size_t _dictionary_width = 0; + size_t _fixed_offset = 0; + size_t _binary_offset = 0; + size_t _index_offset = 0; + uint64_t _dictionary_generation = 0; + size_t _dictionary_decode_calls = 0; +}; + +TEST(DataTypeSerDeParquetTest, MaterializesLogicalUnsignedIntegersDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 0, 7}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 32, + .logical_integer_is_signed = false}; + ParquetMaterializationState state; + DataTypeInt64 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 4294967295LL); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 7); +} + +TEST(DataTypeSerDeParquetTest, NonStrictNarrowUnsignedIntegersPreserveBitWidthCompatibility) { + TestParquetDecodeSource source; + source.set_fixed_values( + {0, 255, 32767, 65535, std::numeric_limits::max(), -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = false}; + IColumn::Filter null_map(6, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt16 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 6, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data, (ColumnInt16::Container {0, 255, 255, 255, 255, 255})); + EXPECT_EQ(null_map, IColumn::Filter(6, 0)); +} + +TEST(DataTypeSerDeParquetTest, LogicalIntegerRejectsOutOfRangePhysicalCarrier) { + TestParquetDecodeSource source; + source.set_fixed_values({1000}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + ParquetMaterializationState state; + state.enable_strict_mode = true; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .is()); + EXPECT_EQ(column->size(), 0); +} + +TEST(DataTypeSerDeParquetTest, NonStrictLogicalIntegerDictionaryPreservesBitWidthCompatibility) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {-24, -24})); + EXPECT_EQ(null_map, IColumn::Filter({0, 0})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFloat16Directly) { + TestParquetDecodeSource source; + source.set_fixed_values({0x3C00, 0xC000, 0x7C00}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::FLOAT16, + .type_length = 2, + .logical_float16 = true}; + ParquetMaterializationState state; + DataTypeFloat32 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(data[0], 1.0F); + EXPECT_FLOAT_EQ(data[1], -2.0F); + EXPECT_TRUE(std::isinf(data[2])); +} + +TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x04, 0xD2, 0xFB, 0x2E}, 2); // 12.34 and -12.34 at scale 2. + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 2, + .decimal_precision = 4, + .decimal_scale = 2}; + ParquetMaterializationState state; + DataTypeDecimal64 type(18, 3); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 12340); + EXPECT_EQ(data[1].value, -12340); +} + +TEST(DataTypeSerDeParquetTest, RescalesExactIntegerDecimalsWithoutRowFailures) { + TestParquetDecodeSource source; + source.set_fixed_values({12300, -98700, 2147483600}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 10, + .decimal_scale = 4}; + ParquetMaterializationState state; + DataTypeDecimal64 type(9, 2); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + // Exact scale reduction can stay on the fast integer path without per-row failure bookkeeping. + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, -987); + EXPECT_EQ(data[2].value, 21474836); +} + +TEST(DataTypeSerDeParquetTest, DecimalScaleDownRejectsLossyPlainAndDictionaryValues) { + ParquetDecodeContext plain_context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 6, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + { + TestParquetDecodeSource source; + source.set_fixed_values({1230, 1234}); + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, 0); + EXPECT_EQ(null_map, IColumn::Filter({0, 1})); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({1234}); + ParquetMaterializationState state; + state.enable_strict_mode = true; + auto column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_dictionary({1230, 1234}, {1, 0, 1}); + auto dictionary_context = plain_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde() + ->read_column_from_parquet(*column, source, dictionary_context, 3, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[1].value, 123); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalUsesWideIntermediateBeforeNarrowing) { + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 19, + .decimal_scale = 0}; + DataTypeDecimal32 type(9, 0); + constexpr int64_t WRAPS_TO_ONE_IN_INT32 = 4294967297LL; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary({WRAPS_TO_ONE_IN_INT32}, {0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values({WRAPS_TO_ONE_IN_INT32}); + context.encoding = ParquetValueEncoding::PLAIN; + } + IColumn::Filter null_map(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1})); + EXPECT_EQ(assert_cast(*column).get_data()[0].value, 0); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalAcceptsSignExtendedBinaryAfterWideExactScaling) { + const std::vector values {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xCE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFB, 0x32}; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 8, + .decimal_precision = 19, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_dictionary(values, 8, {1, 0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_bytes(values, 8); + context.encoding = ParquetValueEncoding::PLAIN; + } + ParquetMaterializationState state; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, dictionary ? -123 : 123); + EXPECT_EQ(data[1].value, dictionary ? 123 : -123); + } +} + +TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { + TestParquetDecodeSource source; + std::vector dictionary {'a', 'a', 'b', 'b'}; + source.set_dictionary(std::move(dictionary), 2, {1, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2}; + ParquetMaterializationState state; + DataTypeString type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + ASSERT_EQ(state.typed_dictionary->size(), 2); + EXPECT_EQ(column->get_data_at(0).to_string(), "bb"); + EXPECT_EQ(column->get_data_at(1).to_string(), "aa"); + EXPECT_EQ(column->get_data_at(2).to_string(), "bb"); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 1); + EXPECT_EQ(column->get_data_at(3).to_string(), "aa"); + + source.set_dictionary({'c', 'c'}, 2, {0}); + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 2); + EXPECT_EQ(column->get_data_at(4).to_string(), "cc"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDictionaryIndicesWithoutValues) { + TestParquetDecodeSource source; + source.set_dictionary({'a', 'a', 'b', 'b'}, 2, {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2, + .dictionary_index_only = true}; + ParquetMaterializationState state; + DataTypeString type; + auto column = ColumnInt32::create(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = column->get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 1); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 1); + EXPECT_FALSE(state.typed_dictionary); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDateDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, 1, -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ParquetMaterializationState state; + DataTypeDateV2 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 2), "1969-12-31"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, -1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1969-12-31 23:59:59.999999"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { + struct TimeUnitCase { + ParquetTimeUnit unit; + int64_t units_per_day; + int64_t expected_last_micros; + }; + const std::vector cases { + {ParquetTimeUnit::MILLIS, 86400000, 86399999000}, + {ParquetTimeUnit::MICROS, 86400000000, 86399999999}, + {ParquetTimeUnit::NANOS, 86400000000000, 86399999999}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.unit)); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = test_case.unit}; + DataTypeTimeV2 type(6); + + TestParquetDecodeSource source; + source.set_fixed_values({0, test_case.units_per_day - 1}); + ParquetMaterializationState state; + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], 0); + EXPECT_EQ(data[1], test_case.expected_last_micros); + + TestParquetDecodeSource nullable_source; + nullable_source.set_fixed_values({-1, test_case.units_per_day}); + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState nullable_state; + nullable_state.conversion_failure_null_map = &null_map; + auto nullable_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*nullable_column, nullable_source, context, + 2, nullable_state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + + TestParquetDecodeSource strict_source; + strict_source.set_fixed_values({-1}); + ParquetMaterializationState strict_state; + auto strict_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*strict_column, strict_source, context, 1, + strict_state) + .ok()); + EXPECT_EQ(strict_column->size(), 0); + + TestParquetDecodeSource unused_invalid_dictionary; + unused_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day - 1, test_case.units_per_day}, {1, 2}); + auto dictionary_context = context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState dictionary_state; + auto dictionary_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, + unused_invalid_dictionary, + dictionary_context, 2, dictionary_state) + .ok()); + EXPECT_EQ(dictionary_column->size(), 2); + + TestParquetDecodeSource referenced_invalid_dictionary; + referenced_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day}, {0}); + ParquetMaterializationState invalid_dictionary_state; + auto invalid_dictionary_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet( + *invalid_dictionary_column, referenced_invalid_dictionary, + dictionary_context, 1, invalid_dictionary_state) + .ok()); + EXPECT_EQ(invalid_dictionary_column->size(), 0); + } +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1969); + EXPECT_EQ(data[0].microsecond(), 999999); + EXPECT_EQ(data[1].second(), 1); + EXPECT_EQ(data[1].microsecond(), 1); +} + +TEST(DataTypeSerDeParquetTest, ValidatesInt96BeforeDateTimeMaterialization) { + { + TestParquetDecodeSource source; + source.set_fixed_values( + {{0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY - 1, JULIAN_UNIX_EPOCH}}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 23:59:59.999999"); + } + + const std::vector invalid_values { + {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}, + {-1, JULIAN_UNIX_EPOCH}, + {0, std::numeric_limits::max()}}; + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { + const std::vector dictionary_values { + {0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + std::vector dictionary(sizeof(TestParquetInt96Timestamp) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(TestParquetInt96Timestamp), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 00:00:00.000000"); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { + constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + + { + TestParquetDecodeSource source; + source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1); + EXPECT_EQ(data[1].year(), 9999); + EXPECT_EQ(data[1].microsecond(), 999999); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max(), YEAR_10000_MILLIS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDecodedValuesUseTheSameBounds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector values {std::numeric_limits::max(), YEAR_10000_MILLIS, 0}; + NullMap conversion_failures(3, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MILLIS, + .row_count = 3, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(conversion_failures, NullMap({1, 1, 0})); + EXPECT_EQ(assert_cast(*column).get_data()[2].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDictionaryFailuresFollowDecodedIds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector dictionary_values {0, YEAR_10000_MILLIS}; + std::vector dictionary(sizeof(int64_t) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(int64_t), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(assert_cast(*column).get_data()[1].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, Int96DecodedValuesRejectInvalidNanos) { + const std::vector values {{NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + NullMap conversion_failures(1, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT96, + .row_count = 1, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(conversion_failures, NullMap({1})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F}, + 16); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::UUID, + .type_length = 16, + .logical_uuid = true}; + ParquetMaterializationState state; + DataTypeVarbinary type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", + 16)); +} + +TEST(DataTypeSerDeParquetTest, NullableConversionFailuresBecomeNullOutsideStrictMode) { + auto expect_null = [](DataTypeSerDeSPtr serde, MutableColumnPtr column, + TestParquetDecodeSource* source, const ParquetDecodeContext& context) { + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + EXPECT_TRUE(serde->read_column_from_parquet(*column, *source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + }; + + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::min()}); + DataTypeDateV2 type; + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeDateTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({10}); + DataTypeDecimal32 type(1, 0); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 2, + .decimal_scale = 0}); + } +} + +TEST(DataTypeSerDeParquetTest, DictionaryConversionFailuresFollowDecodedIds) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map[0], 1); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(source.dictionary_decode_calls(), 1); +} + +TEST(DataTypeSerDeParquetTest, NonNullableDictionaryConversionFailureRemainsAnError) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ParquetMaterializationState state; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 0); +} + +} // namespace +} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 0f0360874c2dd0..97591acf746159 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -629,6 +629,14 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesScannedFromRemote")->value(), 13); EXPECT_EQ(profile.get_counter("BytesScannedFromPeer")->value(), 17); EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + ASSERT_TRUE(children.contains("IO")); + EXPECT_TRUE(children.at("IO").contains("FileCache")); } TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { @@ -650,6 +658,18 @@ TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); } +TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) { + auto status = FileScannerV2::TEST_contextualize_output_filter_status( + Status::InvalidArgument("synthetic row filter failure"), TFileFormatType::FORMAT_ORC); + EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; + EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; + + status = FileScannerV2::TEST_contextualize_output_filter_status( + Status::InvalidArgument("synthetic row filter failure"), + TFileFormatType::FORMAT_PARQUET); + EXPECT_EQ(status.to_string().find("nextBatch failed"), std::string::npos) << status; +} + // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index e134248f814081..4bb7b920231a68 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -414,10 +414,8 @@ class ExecutableStructElementExpr final : public VExpr { Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { ColumnPtr struct_column; - // branch-4.1's public execution API still takes a mutable selector even though expression - // implementations receive it as read-only; this test expression only forwards it. - RETURN_IF_ERROR(get_child(0)->execute_column( - context, block, const_cast(selector), count, struct_column)); + RETURN_IF_ERROR( + get_child(0)->execute_column(context, block, selector, count, struct_column)); const auto& input = assert_cast(*struct_column); result_column = input.get_column_ptr(0); return Status::OK(); @@ -619,7 +617,7 @@ class Int64ChildGreaterThanExpr final : public VExpr { size_t count, ColumnPtr& result_column) const override { ColumnPtr child_column; RETURN_IF_ERROR( - get_child(0)->execute_column_impl(context, block, selector, count, child_column)); + get_child(0)->execute_column(context, block, selector, count, child_column)); const auto& input = assert_cast(*child_column); auto result = ColumnUInt8::create(); auto& result_data = result->get_data(); @@ -655,11 +653,10 @@ class Int64BinaryPredicateExpr final : public VExpr { Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { ColumnPtr left_column; - RETURN_IF_ERROR( - get_child(0)->execute_column_impl(context, block, selector, count, left_column)); + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left_column)); ColumnPtr right_column; RETURN_IF_ERROR( - get_child(1)->execute_column_impl(context, block, selector, count, right_column)); + get_child(1)->execute_column(context, block, selector, count, right_column)); auto result = ColumnUInt8::create(); auto& result_data = result->get_data(); @@ -2488,10 +2485,41 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(1)})); ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } +TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsPayloadForScannerBoundary) { + const auto int_type = i32(); + auto quantity = name_col("ss_quantity", int_type); + auto tax = name_col("ss_ext_tax", int_type); + const std::vector table_schema = {quantity, tax}; + const std::vector file_schema = { + name_col("ss_quantity", int_type, 0), + name_col("ss_ext_tax", int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + + auto filter_expr = int_gt(table_slot(7, 0, int_type, "ss_quantity"), 20); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request).ok()); + + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); + // The scanner evaluates its table-level conjuncts after TableReader returns, so a visible + // predicate slot cannot be replaced with a default-valued placeholder at the file boundary. + EXPECT_TRUE(request.predicate_only_columns.empty()); +} + TEST(ColumnMapperScanRequestTest, StructOutputAndFilterOnlyChildAreMerged) { const auto int_type = i32(); const auto string_type = str(); @@ -3013,6 +3041,7 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(10)})); EXPECT_TRUE(request.predicate_columns[0].project_all_children); EXPECT_TRUE(request.predicate_columns[0].children.empty()); @@ -3938,116 +3967,6 @@ TEST_F(ColumnMapperCastTest, ColumnMapperCastsLiteralForLiteralSlotPredicateType file_request.conjuncts[0]->close(); } -// Scenario: Nereids may represent `BIGINT value = 1` as `value = CAST(INT 1 AS BIGINT)`. -// Strip a provably lossless literal widening so metadata readers can recognize slot-literal -// predicates for zone-map pruning. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyCastLiteral) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto table_column = name_col("value", i64()); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", i64(), 0); - std::vector file_schema {file_field}; - - auto status = mapper.create_mapping(projected_columns, {}, file_schema); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::GT); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child(cast_expr(VLiteral::create_shared(i32(), Field::create_field(1)), - table_column.type)); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); - - Block block; - block.insert(ColumnHelper::create_column_with_name({1, 2})); - auto* conjunct = file_request.conjuncts[0].get(); - ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok()); - ASSERT_TRUE(conjunct->open(&state).ok()); - IColumn::Filter filter(block.rows(), 1); - bool can_filter_all = false; - ASSERT_TRUE( - conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all) - .ok()); - EXPECT_EQ(filter, IColumn::Filter({0, 1})); - conjunct->close(); -} - -// Scenario: Nereids may keep a narrow literal directly under an implicitly coerced comparison, -// for example `nullable INT value = TINYINT 1`. Normalize the literal to the table type so the -// file predicate remains recognizable to Parquet zone-map pruning. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyTypedLiteral) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto column_type = make_nullable(i32()); - auto table_column = name_col("value", column_type); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", column_type, 0); - std::vector file_schema {file_field}; - - auto status = mapper.create_mapping(projected_columns, {}, file_schema); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::GT); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child( - VLiteral::create_shared(std::make_shared(), - Field::create_field(static_cast(1)))); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); -} - -// Scenario: branch-4.1 Nereids comparisons can be resolved by function name without retaining the -// legacy opcode. The node kind still identifies a binary comparison, so its narrow literal must be -// normalized exactly like the opcode-bearing form used above. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesBinaryPredicateWithoutLegacyOpcode) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto column_type = make_nullable(i32()); - auto table_column = name_col("value", column_type); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", column_type, 0); - - auto status = mapper.create_mapping(projected_columns, {}, {file_field}); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::INVALID_OPCODE); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child( - VLiteral::create_shared(std::make_shared(), - Field::create_field(static_cast(1)))); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[0]->data_type()->equals(*file_field.type)); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); -} - // Scenario: a fractional table literal cannot be localized to an integral file type without // changing the predicate boundary, so the mapper must cast the file slot instead. TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyBinaryLiteralConversion) { diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp index 0ce99630c7af8b..c04e17e07f64b8 100644 --- a/be/test/format_v2/delimited_text/csv_reader_test.cpp +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -446,7 +446,7 @@ TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); @@ -755,6 +755,20 @@ TEST_F(CsvV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(CsvV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: CSV v2 parses enclosed fields itself instead of delegating to the old CsvReader. A // separator inside an enclosed string must stay inside the same CSV field. TEST_F(CsvV2ReaderTest, EnclosedFieldKeepsSeparatorInsideStringValue) { diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp index f5f7309e490ff1..4927ee7622e4d4 100644 --- a/be/test/format_v2/delimited_text/text_reader_test.cpp +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -41,6 +41,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_profile.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "testutil/desc_tbl_builder.h" @@ -414,7 +415,7 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); @@ -423,6 +424,33 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); } +TEST_F(TextV2ReaderTest, FormatProfilesKeepDistinctCountersInBothInitializationOrders) { + auto expect_format_children = [](RuntimeProfile* profile) { + TRuntimeProfileTree tree; + profile->to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("DelimitedTextReader")); + EXPECT_TRUE(children.at("DelimitedTextReader").contains("DelimitedRowsFilteredByConjunct")); + EXPECT_FALSE(children.at("DelimitedTextReader").contains("RowsFilteredByConjunct")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowsFilteredByConjunct")); + EXPECT_FALSE(children.at("ParquetReader").contains("DelimitedRowsFilteredByConjunct")); + }; + + RuntimeProfile parquet_first("parquet_first"); + parquet::ParquetProfile parquet_first_counters; + parquet_first_counters.init(&parquet_first); + auto parquet_first_text = create_reader(_file_path, &_params, _slots, &_state, &parquet_first); + expect_format_children(&parquet_first); + + RuntimeProfile text_first("text_first"); + auto text_first_reader = create_reader(_file_path, &_params, _slots, &_state, &text_first); + parquet::ParquetProfile text_first_parquet_counters; + text_first_parquet_counters.init(&text_first); + expect_format_children(&text_first); +} + // Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic // children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the // slot type while keeping the top-level local id as the text field ordinal from column_idxs. @@ -909,6 +937,20 @@ TEST_F(TextV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(TextV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: a non-first split starts inside a text record and must skip the partial first line. TEST_F(TextV2ReaderTest, NonFirstSplitSkipsPartialFirstRecord) { const auto split_path = (_test_dir / "split.text").string(); diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index 7feebdca113dcc..95eda4a557e790 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -19,9 +19,12 @@ #include +#include #include #include +#include #include +#include #include #include "io/io_common.h" @@ -39,9 +42,15 @@ class FakeJniTableReader final : public JniTableReader { std::vector propagated_batch_sizes; std::vector close_results; bool next_eof = false; + std::chrono::milliseconds init_delay {0}; + std::chrono::milliseconds open_delay {0}; + std::chrono::milliseconds close_delay {0}; protected: - std::string connector_class() const override { return "test/FakeJniScanner"; } + std::string connector_class() const override { + std::this_thread::sleep_for(init_delay); + return "test/FakeJniScanner"; + } Status build_scanner_params(std::map* params) const override { params->clear(); @@ -56,6 +65,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _close_jni_scanner() override { + std::this_thread::sleep_for(close_delay); if (!TEST_scanner_opened()) { return Status::OK(); } @@ -79,6 +89,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _open_jni_scanner() override { + std::this_thread::sleep_for(open_delay); open_batch_sizes.push_back(TEST_batch_size()); TEST_set_split_state(true, false); return Status::OK(); @@ -177,6 +188,7 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { .partition_values = {}, .conjuncts = std::nullopt, .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, .condition_cache_digest = std::nullopt, .cache = nullptr, .current_range = {}, @@ -189,5 +201,36 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { + constexpr auto delay = std::chrono::milliseconds(8); + RuntimeProfile profile("JniLifecycleContainment"); + FakeJniTableReader reader; + reader.init_delay = delay; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + ASSERT_GE(profile.get_counter("InitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.open_delay = delay; + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + ASSERT_GE(profile.get_counter("PrepareSplitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.close_delay = delay; + ASSERT_TRUE(reader.close().ok()); + EXPECT_GE(profile.get_counter("CloseTime")->value(), + std::chrono::duration_cast(delay).count()); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp index 54c89505edc3f5..8d63a4ab21ae65 100644 --- a/be/test/format_v2/orc/orc_file_input_stream_test.cpp +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -329,14 +329,49 @@ TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { input.beforeReadStripe( std::make_unique(200, std::vector {}), selected_columns({}), next_streams); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); } - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); +} + +TEST(OrcFileInputStreamTest, MergedIoChildrenStayIsolatedInBothInitializationOrders) { + for (const bool generic_first : {false, true}) { + auto reader = std::make_shared(256); + RuntimeProfile profile(generic_first ? "generic_first" : "orc_first"); + auto register_generic = [&] { + ADD_TIMER(&profile, "MergedSmallIO"); + return ADD_CHILD_COUNTER_WITH_LEVEL(&profile, "RequestIO", TUnit::UNIT, "MergedSmallIO", + 1); + }; + RuntimeProfile::Counter* generic_request_io = nullptr; + if (generic_first) { + generic_request_io = register_generic(); + } + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + if (!generic_first) { + generic_request_io = register_generic(); + } + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + std::array data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + } + ASSERT_NE(generic_request_io, nullptr); + EXPECT_EQ(generic_request_io->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 1); + } } } // namespace diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 6d41c98534eb38..63a09b04b1cced 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -62,17 +62,18 @@ #include "core/data_type/primitive_type.h" #include "core/value/timestamptz_value.h" #include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/orc/orc_memory_stream_test.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -359,6 +360,21 @@ class NullableInt32GreaterThanExpr final : public VExpr { const std::string _expr_name = "NullableInt32GreaterThanExpr"; }; +class FailingRowFilterExpr final : public VExpr { +public: + FailingRowFilterExpr() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InvalidArgument("synthetic row filter failure"); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _expr_name = "FailingRowFilterExpr"; +}; + class NullableInt32LessThanExpr final : public VExpr { public: NullableInt32LessThanExpr(int column_id, int32_t value) @@ -618,8 +634,8 @@ class CompoundPredicateExpr final : public VExpr { if (_opcode == TExprOpcode::COMPOUND_NOT) { DORIS_CHECK(_children.size() == 1); ColumnPtr child_column; - RETURN_IF_ERROR(_children[0]->execute_column_impl(context, block, selector, count, - child_column)); + RETURN_IF_ERROR( + _children[0]->execute_column(context, block, selector, count, child_column)); for (size_t row = 0; row < count; ++row) { result_data[row] = !bool_value(*child_column, row); } @@ -633,8 +649,7 @@ class CompoundPredicateExpr final : public VExpr { child_columns.reserve(_children.size()); for (const auto& child : _children) { ColumnPtr child_column; - RETURN_IF_ERROR( - child->execute_column_impl(context, block, selector, count, child_column)); + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, child_column)); child_columns.push_back(std::move(child_column)); } for (size_t row = 0; row < count; ++row) { @@ -688,30 +703,9 @@ class RuntimeFilterWrapperExpr final : public VExpr { VExprSPtr get_impl() const override { return _impl; } - const VExprSPtrs& children() const override { return _impl->children(); } - - TExprNodeType::type node_type() const override { return _impl->node_type(); } - - Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { - RETURN_IF_ERROR(_impl->prepare(state, desc, context)); - _prepare_finished = true; - return Status::OK(); - } - - Status open(RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope) override { - RETURN_IF_ERROR(_impl->open(state, context, scope)); - _open_finished = true; - return Status::OK(); - } - - void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override { - _impl->close(context, scope); - } - Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { - return _impl->execute_column_impl(context, block, selector, count, result_column); + return _impl->execute_column(context, block, selector, count, result_column); } const std::string& expr_name() const override { return _expr_name; } @@ -4492,8 +4486,6 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t class NewOrcReaderTest : public testing::Test { protected: - static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); } - enum class DirectInPredicateShape { ROOT, AND, @@ -4589,7 +4581,7 @@ class NewOrcReaderTest : public testing::Test { auto node = make_filter_in_node(TExprNodeType::IN_PRED); auto direct_in = VDirectInPredicate::create_shared(node, filter, true); direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); - VExprSPtr predicate = std::make_shared(direct_in); + VExprSPtr predicate = RuntimeFilterExpr::create_shared(node, direct_in, 0.0, false, 7); if (shape == DirectInPredicateShape::AND) { predicate = std::make_shared( TExprOpcode::COMPOUND_AND, @@ -5286,16 +5278,17 @@ TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { } ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); - ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); - ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); - EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); - EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); - EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); - EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); - EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedOverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("OrcMergedRequestIO")->value(), + profile.get_counter("OrcMergedIO")->value()); + EXPECT_GT(profile.get_counter("OrcMergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OrcMergedOverReadBytes")->value(), 0); } TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { @@ -5686,6 +5679,29 @@ TEST_F(NewOrcReaderTest, ConjunctFiltersRows) { EXPECT_EQ(values.get_data_at(2).to_string(), "five"); } +TEST_F(NewOrcReaderTest, RowFilterFailureRetainsNextBatchContext) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared())); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; + EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; +} + TEST_F(NewOrcReaderTest, OrcLazyDecodesOnlySelectedNonPredicateRows) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -5746,8 +5762,8 @@ TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 2); } TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { @@ -5776,8 +5792,8 @@ TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); } TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { @@ -6725,13 +6741,13 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { ASSERT_EQ(result_rows, 200); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); - ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcFileNum"), nullptr); const std::array orc_reader_metric_counters { "ReaderCall", "ReaderInclusiveLatencyUs", @@ -6750,19 +6766,63 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { for (const auto counter_name : orc_reader_metric_counters) { ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; } - EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsReadNum")->value(), 1); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); - EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); - EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); - EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("OrcFilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcFileNum")->value(), 1); EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), static_cast(file_reader_stats.read_rows)); } +TEST_F(NewOrcReaderTest, ProfileKeepsPruningCountersBelowOrcReader) { + RuntimeProfile profile("new_orc_reader_hierarchy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("OrcReader")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsFiltered")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsReadNum")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcFilteredRowsByGroup")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenParquetInitializesFirst) { + RuntimeProfile profile("parquet_then_orc_profile"); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenOrcInitializesFirst) { + RuntimeProfile profile("orc_then_parquet_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); @@ -6799,10 +6859,10 @@ TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 0); } TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { @@ -7055,7 +7115,7 @@ TEST_F(NewOrcReaderTest, SargNullAwareRuntimeFilterDoesNotPruneNullStripe) { auto node = make_filter_in_node(TExprNodeType::NULL_AWARE_IN_PRED); auto direct_in = VDirectInPredicate::create_shared(node, filter, true); direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); - auto runtime_filter = VRuntimeFilterWrapper::create_shared(node, direct_in, 0.0, true, 7); + auto runtime_filter = RuntimeFilterExpr::create_shared(node, direct_in, 0.0, true, 7); auto runtime_filter_context = VExprContext::create_shared(std::move(runtime_filter)); ASSERT_TRUE(runtime_filter_context->prepare(&state, RowDescriptor()).ok()); ASSERT_TRUE(runtime_filter_context->open(&state).ok()); @@ -9289,6 +9349,7 @@ TEST_F(NewOrcReaderTest, SargTimestampInstantRepeatedCivilTimeDoesNotPruneStripe auto reader = create_reader_for_path(multi_stripe_file_path); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TimezoneUtils::load_timezones_to_cache(); state.set_timezone("America/New_York"); ASSERT_TRUE(reader->init(&state).ok()); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp new file mode 100644 index 00000000000000..1cf6fdaade7cc4 --- /dev/null +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -0,0 +1,3773 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/column_reader.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader.h" +#include "util/block_compression.h" +#include "util/coding.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/thrift_util.h" +#include "util/timezone_utils.h" + +namespace doris::format::parquet::native { +namespace { + +VExprSPtr create_int32_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t literal, + bool literal_on_left = false) { + const auto int_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(column_id, column_id, -1, int_type, "plain_int"); + auto value = VLiteral::create_shared(int_type, Field::create_field(literal)); + if (literal_on_left) { + root->add_child(value); + root->add_child(slot); + } else { + root->add_child(slot); + root->add_child(value); + } + return root; +} + +VExprSPtr create_float_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, float literal) { + const auto float_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({float_type->to_thrift(), float_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + root->add_child(VSlotRef::create_shared(column_id, column_id, -1, float_type, "plain_float")); + root->add_child(VLiteral::create_shared(float_type, Field::create_field(literal))); + return root; +} + +TEST(ParquetV2NativeDecoderTest, RawExprEvaluatesPhysicalValuesWithoutColumn) { + const std::array values = {1, 4, 7, 10, 13, 16}; + std::array matches {}; + matches.fill(1); + const auto greater_equal = create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 7); + const auto less_than = create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 16); + const auto int_type = std::make_shared(); + + ASSERT_TRUE(greater_equal->can_execute_on_raw_fixed_values(int_type, 0)); + ASSERT_TRUE(greater_equal + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + ASSERT_TRUE(less_than + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); + + matches.fill(1); + const auto literal_less_than_slot = + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 7, true); + ASSERT_TRUE(literal_less_than_slot + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawExprPreservesFloatNanOrdering) { + const float nan = std::numeric_limits::quiet_NaN(); + const std::array values {-1, 0, 1, nan}; + const auto float_type = std::make_shared(); + + std::array matches {}; + matches.fill(1); + const auto greater_than_zero = create_float_raw_comparison(0, "gt", TExprOpcode::GT, 0); + ASSERT_TRUE(greater_than_zero + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1})); + + matches.fill(1); + const auto equals_nan = create_float_raw_comparison(0, "eq", TExprOpcode::EQ, nan); + ASSERT_TRUE(equals_nan + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawFixedFilterSupportsIdentityWidthEncodingTypes) { + using Reader = ColumnChunkReader; + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT64)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::FLOAT)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::DOUBLE)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT64)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::FLOAT)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::RLE_DICTIONARY, + tparquet::Type::INT32)); +} + +class RejectFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return Status::InternalError("Unexpected fixed dictionary"); + } +}; + +class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + refs.assign(values, values + num_values); + return Status::OK(); + } + + std::vector refs; +}; + +class CapturePlainBinaryLayoutConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + legacy_consume_called = true; + return Status::InternalError("PLAIN BYTE_ARRAY used the legacy StringRef path"); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + base = encoded_data; + source_offsets.assign(payload_offsets, payload_offsets + num_values); + output_offsets.assign(value_offsets, value_offsets + num_values + 1); + spans = value_spans; + return Status::OK(); + } + + const char* base = nullptr; + bool legacy_consume_called = false; + std::vector source_offsets; + std::vector output_offsets; + std::vector spans; +}; + +class CaptureFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + ++consume_calls; + if (width == 0) { + width = value_width; + } + DORIS_CHECK_EQ(width, value_width); + bytes.insert(bytes.end(), values, values + num_values * value_width); + return Status::OK(); + } + + template + std::vector values() const { + DORIS_CHECK_EQ(width, sizeof(T)); + DORIS_CHECK_EQ(bytes.size() % sizeof(T), 0); + std::vector result(bytes.size() / sizeof(T)); + memcpy(result.data(), bytes.data(), bytes.size()); + return result; + } + + size_t width = 0; + size_t consume_calls = 0; + std::vector bytes; +}; + +TEST(ParquetV2NativeDecoderTest, FragmentedPlainSelectionUsesOneConsumerBatch) { + std::array input {}; + std::iota(input.begin(), input.end(), 0); + Slice data(reinterpret_cast(input.data()), input.size() * sizeof(int32_t)); + FixLengthPlainDecoder decoder; + decoder.set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder.set_data(&data).ok()); + + ParquetSelection selection { + .total_values = input.size(), .selected_values = input.size() / 2, .ranges = {}}; + for (size_t row = 0; row < input.size(); row += 2) { + selection.ranges.push_back({.first = row, .count = 1}); + } + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder.decode_selected_fixed_values(selection, consumer).ok()); + + EXPECT_EQ(consumer.consume_calls, 1); + std::vector expected; + for (int32_t value = 0; value < static_cast(input.size()); value += 2) { + expected.push_back(value); + } + EXPECT_EQ(consumer.values(), expected); +} + +class ScriptedDictionaryMaterializationSource final : public ParquetDecodeSource { +public: + ScriptedDictionaryMaterializationSource(std::vector ids, bool prefer_indices) + : _ids(std::move(ids)), _prefer_indices(prefer_indices) {} + + Status decode_fixed_values(size_t, ParquetFixedValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no fixed values"); + } + + Status decode_binary_values(size_t, ParquetBinaryValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no binary values"); + } + + Status skip_values(size_t) override { return Status::OK(); } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++index_batches; + *indices = _ids; + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++direct_batches; + return consumer.consume_indices(_ids.data(), _ids.size()); + } + + bool prefer_dictionary_index_materialization(size_t) const override { return _prefer_indices; } + + size_t direct_batches = 0; + size_t index_batches = 0; + +private: + std::vector _ids; + bool _prefer_indices; +}; + +const RowRanges& scripted_row_ranges() { + static const RowRanges ranges; + return ranges; +} + +class ScriptedColumnReader final : public ColumnReader { +public: + ScriptedColumnReader(size_t rows, size_t values, bool eof, std::vector rep_levels, + std::vector def_levels) + : ColumnReader(scripted_row_ranges(), rows, nullptr, nullptr), + _rows(rows), + _values(values), + _eof(eof), + _rep_levels(std::move(rep_levels)), + _def_levels(std::move(def_levels)) {} + + Status read_column_data(ColumnPtr& column, const DataTypePtr&, + const std::shared_ptr&, FilterMap&, size_t, + size_t* read_rows, bool* eof, bool, int64_t = -1) override { + if (_used) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + column = IColumn::mutate(std::move(column)); + column->assert_mutable()->insert_many_defaults(_values); + *read_rows = _rows; + *eof = _eof; + _used = true; + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t* read_rows, bool* eof) override { + *read_rows = _rows; + *eof = _eof; + return Status::OK(); + } + + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { return {}; } + void close() override {} + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override {} + +private: + size_t _rows; + size_t _values; + bool _eof; + bool _used = false; + std::vector _rep_levels; + std::vector _def_levels; +}; + +class MemoryBufferedReader final : public io::BufferedStreamReader { +public: + explicit MemoryBufferedReader(std::vector data) : _data(std::move(data)) {} + + Status read_bytes(const uint8_t** buf, uint64_t offset, size_t bytes_to_read, + const io::IOContext*) override { + if (offset > _data.size() || bytes_to_read > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + *buf = _data.data() + offset; + return Status::OK(); + } + Status read_bytes(Slice& slice, uint64_t offset, const io::IOContext*) override { + if (offset > _data.size() || slice.size > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + slice.data = reinterpret_cast(_data.data() + offset); + return Status::OK(); + } + std::string path() override { return "memory.parquet"; } + int64_t mtime() const override { return 0; } + +private: + std::vector _data; +}; + +class NativeDecoderMemoryFileReader final : public io::FileReader { +public: + explicit NativeDecoderMemoryFileReader(std::vector data) + : _data(std::move(data)), _path("native-decoder-memory.parquet") {} + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _data.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _data.size() || result.size > _data.size() - offset) { + return Status::IOError("native decoder memory read exceeds file"); + } + memcpy(result.data, _data.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); + } + +private: + std::vector _data; + io::Path _path; + bool _closed = false; +}; + +std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { + auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, + physical_type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector& values, + int32_t* length) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + *length = static_cast(total_size); + auto dictionary = make_unique_buffer(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(dictionary.get() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(dictionary.get() + offset, value.data(), value.size()); + offset += value.size(); + } + return dictionary; +} + +std::vector encode_plain_byte_arrays(const std::vector& values) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + std::vector encoded(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(encoded.data() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(encoded.data() + offset, value.data(), value.size()); + offset += value.size(); + } + return encoded; +} + +std::vector serialize_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physical_type, + tparquet::Encoding::type encoding, bool all_null) { + constexpr size_t VALUE_COUNT = 3; + const std::vector encoded_levels {static_cast(VALUE_COUNT << 1), + static_cast(all_null ? 0 : 1)}; + std::vector payload; + tparquet::PageHeader header; + if (data_page_v2) { + payload = encoded_levels; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(VALUE_COUNT); + header.data_page_header_v2.__set_num_nulls(all_null ? VALUE_COUNT : 0); + header.data_page_header_v2.__set_num_rows(VALUE_COUNT); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_definition_levels_byte_length(encoded_levels.size()); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + header.type = tparquet::PageType::DATA_PAGE; + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(VALUE_COUNT); + header.data_page_header.__set_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(VALUE_COUNT); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BOOLEAN) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.definition_level = 1; + field.parquet_schema.__set_type(physical_type); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, VALUE_COUNT, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + const auto load_status = chunk_reader.load_page_data(); + EXPECT_TRUE(load_status.ok()) << load_status; + RETURN_IF_ERROR(load_status); + + level_t level = -1; + EXPECT_EQ(chunk_reader.def_level_decoder().get_next_run(&level, VALUE_COUNT), VALUE_COUNT); + EXPECT_EQ(level, all_null ? 0 : 1); + FilterMap filter; + RETURN_IF_ERROR(filter.init(nullptr, VALUE_COUNT, false)); + NullMap null_map; + ColumnSelectVector select_vector; + const std::vector null_runs {static_cast(all_null ? 0 : VALUE_COUNT), + static_cast(all_null ? VALUE_COUNT : 0)}; + RETURN_IF_ERROR(select_vector.init(null_runs, VALUE_COUNT, &null_map, &filter, 0)); + auto column = field.data_type->create_column(); + ParquetDecodeContext decode_context; + static const auto utc = cctz::utc_time_zone(); + RETURN_IF_ERROR(init_decode_context_for_test(field, &utc, &decode_context)); + ParquetMaterializationState state; + state.enable_strict_mode = true; + const auto status = chunk_reader.materialize_values(column, *field.data_type->get_serde(), + decode_context, state, select_vector); + if (status.ok()) { + EXPECT_EQ(column->size(), VALUE_COUNT); + EXPECT_EQ(null_map, NullMap(VALUE_COUNT, all_null ? 1 : 0)); + } + return status; +} + +Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + tparquet::CompressionCodec::type codec, bool preload_page_cache = false) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + const size_t chunk_size = bytes.size(); + const std::string page_cache_file_key = "native-scripted-page-" + + std::to_string(static_cast(header.type)) + "-" + + std::to_string(header.compressed_page_size) + "-" + + std::to_string(header.uncompressed_page_size); + if (preload_page_cache) { + auto* page = new DataPage(bytes.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), bytes.data(), bytes.size()); + page->reset_size(bytes.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(page_cache_file_key, chunk_size, 0), page, &handle, + segment_v2::DATA_PAGE); + } + MemoryBufferedReader reader(std::move(bytes)); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(codec); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(chunk_size); + if (header.type == tparquet::PageType::DICTIONARY_PAGE) { + chunk.meta_data.__set_dictionary_page_offset(0); + chunk.meta_data.__set_data_page_offset(0); + } else { + chunk.meta_data.__set_data_page_offset(0); + } + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(preload_page_cache, page_cache_file_key); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + return chunk_reader.load_page_data(); +} + +Status load_malformed_nested_page(tparquet::PageHeader header, const std::vector& payload, + int64_t metadata_values = std::numeric_limits::max(), + level_t max_repetition_level = 1) { + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(metadata_values); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = max_repetition_level; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + std::vector rep_levels; + size_t result_rows = 0; + bool cross_page = false; + return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); +} + +Status materialize_plain_int96(const std::vector& values, + const std::vector& filter_values = {}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__set_uncompressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values.size()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page( + header, std::vector(reinterpret_cast(values.data()), + reinterpret_cast(values.data()) + + values.size() * sizeof(ParquetInt96Timestamp))); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT96); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(values.size()); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT96; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, values.size(), + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT96; + static const auto utc = cctz::utc_time_zone(); + decode_context.timezone = &utc; + ParquetMaterializationState state; + state.enable_strict_mode = true; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.empty() ? nullptr : filter_values.data(), + filter_values.size(), false)); + ColumnSelectVector select_vector; + const std::vector run_length_null_map {static_cast(values.size()), 0}; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, values.size(), nullptr, &filter, 0)); + return chunk_reader.materialize_values(column, *type.get_serde(), decode_context, state, + select_vector); +} + +template +Status materialize_selected_plain_fixed( + const std::vector& physical_values, size_t logical_values, + const std::vector& run_length_null_map, const std::vector& filter_values, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const DataType& type, MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics, bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(static_cast(logical_values)); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(PhysicalType)); + if (!payload.empty()) { + memcpy(payload.data(), physical_values.data(), payload.size()); + } + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.enable_strict_mode = strict_mode; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT32, + ParquetPhysicalType::INT32, type, column, null_map, + statistics, strict_mode, context_override); +} + +template +Status materialize_selected_plain_int64(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT64, + ParquetPhysicalType::INT64, type, column, null_map, + statistics, false, &context); +} + +template +Status materialize_selected_dictionary_fixed( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const ParquetDecodeContext* context_override = nullptr) { + std::vector dictionary_payload(dictionary.size() * sizeof(PhysicalType)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + // Real Parquet files place column chunks after the file header. Keep the dictionary offset + // non-zero because zero is the metadata sentinel for "no dictionary page". + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 4); + for (const auto id : physical_ids) { + encoder.Put(id); + } + // Parquet bit-packed dictionary runs declare groups of eight; emit the trailing padding so a + // decoder that buffers the declared run never reads beyond the synthetic page. + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 4; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_dictionary_int32( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics) { + return materialize_selected_dictionary_fixed( + dictionary, physical_ids, logical_values, run_length_null_map, filter_values, type, + column, null_map, statistics, tparquet::Type::INT32, ParquetPhysicalType::INT32); +} + +template +Status materialize_selected_dictionary_int64( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_dictionary_fixed(dictionary, physical_ids, logical_values, + run_length_null_map, filter_values, type, column, + null_map, statistics, tparquet::Type::INT64, + ParquetPhysicalType::INT64, &context); +} + +Status materialize_selected_dictionary_strings(const std::vector& dictionary, + const std::vector& physical_ids, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics) { + auto dictionary_payload = encode_plain_byte_arrays(dictionary); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (const auto id : physical_ids) { + encoder.Put(id); + } + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 2; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeString type; + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::BYTE_ARRAY; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +TEST(ParquetV2NativeDecoderTest, RawExprMapsNullableSparseRowsDirectly) { + const std::vector physical_values {1, 4, 7, 10, 13}; + constexpr size_t LOGICAL_VALUES = 7; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(LOGICAL_VALUES); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + memcpy(payload.data(), physical_values.data(), payload.size()); + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(LOGICAL_VALUES); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, + nullptr, page_context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + + const std::vector null_runs {1, 1, 2, 1, 2}; + const std::vector input_filter {1, 0, 1, 1, 1, 0, 1}; + FilterMap filter; + ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); + const VExprSPtrs predicates {create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 4), + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 13)}; + NullMap selected_nulls; + IColumn::Filter physical_matches; + auto projected_column = make_nullable(field.data_type)->create_column(); + IColumn::Filter row_filter; + bool used_filter = false; + ASSERT_TRUE(chunk_reader + .filter_fixed_width_values(predicates, 0, select_vector, &selected_nulls, + &physical_matches, projected_column.get(), + &row_filter, &used_filter) + .ok()); + + EXPECT_TRUE(used_filter); + EXPECT_EQ(selected_nulls, (NullMap {0, 0, 0, 1, 0})); + EXPECT_EQ(physical_matches, (IColumn::Filter {0, 1, 1, 0})); + EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 0, 0})); + ASSERT_EQ(projected_column->size(), 2); + EXPECT_FALSE(projected_column->is_null_at(0)); + EXPECT_FALSE(projected_column->is_null_at(1)); + const auto& projected_values = + assert_cast( + assert_cast(*projected_column).get_nested_column()) + .get_data(); + EXPECT_EQ(projected_values[0], 4); + EXPECT_EQ(projected_values[1], 7); + EXPECT_EQ(chunk_reader.remaining_num_values(), 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainPrimitiveSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeDecimal32 type(9, 0); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 9, + .decimal_scale = 0}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + const auto& values = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values[0].value, 0); + EXPECT_EQ(values[1].value, 500); + EXPECT_EQ(values[2].value, 0); + EXPECT_EQ(values[3].value, 1000); + EXPECT_EQ(values[4].value, 0); + EXPECT_EQ(null_map, (NullMap {0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateSelectionBatchesPhysicalPayload) { + const std::vector physical_days {0, 1, 2, 3, 4}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateV2 type; + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_days, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-04"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhysicalPayload) { + const std::vector physical_micros {0, 1'000'000, 2'000'000, 3'000'000, 4'000'000}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000000"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-01 00:00:03.000000"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NegativeNanosFloorAcrossPlainAndDictionaryTimestamps) { + const std::vector nanos {-1, -999, -1000, -1001, 0, 1001}; + const std::vector dictionary_ids {0, 1, 2, 3, 4, 5}; + const std::vector no_nulls {static_cast(nanos.size()), 0}; + const std::vector select_all(nanos.size(), 1); + const std::vector expected { + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999999", + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999998", + "1970-01-01 00:00:00.000000", "1970-01-01 00:00:00.000001"}; + const ParquetDecodeContext local_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS}; + const ParquetDecodeContext utc_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS, + .timestamp_is_adjusted_to_utc = true}; + + for (const bool dictionary : {false, true}) { + SCOPED_TRACE(dictionary ? "dictionary" : "plain"); + DataTypeDateTimeV2 datetime_type(6); + auto datetime_column = datetime_type.create_column(); + NullMap datetime_nulls; + ColumnChunkReaderStatistics datetime_statistics; + const auto datetime_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + datetime_type, &datetime_column, &datetime_nulls, + &datetime_statistics, local_context) + : materialize_selected_plain_int64(nanos, nanos.size(), no_nulls, + select_all, datetime_type, + &datetime_column, &datetime_nulls, + &datetime_statistics, local_context); + ASSERT_TRUE(datetime_status.ok()) << datetime_status; + ASSERT_EQ(datetime_column->size(), expected.size()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(datetime_type.to_string(*datetime_column, row), expected[row]); + } + + DataTypeTimeStampTz timestamptz_type(6); + auto timestamptz_column = timestamptz_type.create_column(); + NullMap timestamptz_nulls; + ColumnChunkReaderStatistics timestamptz_statistics; + const auto timestamptz_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + timestamptz_type, ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context) + : materialize_selected_plain_int64( + nanos, nanos.size(), no_nulls, select_all, timestamptz_type, + ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context); + ASSERT_TRUE(timestamptz_status.ok()) << timestamptz_status; + const auto& tz_column = assert_cast(*timestamptz_column); + const auto utc = cctz::utc_time_zone(); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(tz_column.get_element(row).to_string(utc, 6), expected[row] + "+00:00"); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector dictionary(16); + std::iota(dictionary.begin(), dictionary.end(), 100); + // One value followed by nineteen NULLs exercises the sparse-null threshold (<10%). + std::vector null_runs; + for (size_t row = 0; row < LOGICAL_VALUES;) { + null_runs.push_back(1); + ++row; + const auto null_count = std::min(19, LOGICAL_VALUES - row); + null_runs.push_back(cast_set(null_count)); + row += null_count; + } + std::vector physical_ids((LOGICAL_VALUES + 19) / 20); + for (size_t index = 0; index < physical_ids.size(); ++index) { + physical_ids[index] = static_cast(index % dictionary.size()); + } + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + const auto status = materialize_selected_dictionary_int32( + dictionary, physical_ids, LOGICAL_VALUES, null_runs, filter, type, &column, &null_map, + &statistics); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 100, 102, 0, 104, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDictionaryIsClean) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + auto& dictionary = assert_cast(*state.typed_dictionary).get_data(); + dictionary = {10, 20, 30, 40}; + state.dictionary_indices = {3, 0, 2, 1, 3}; + state.dictionary_conversion_failures.resize_fill(dictionary.size(), 0); + + auto output = ColumnInt32::create(); + ASSERT_TRUE(state.materialize_dictionary(*output).ok()); + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_failure_scan_rows, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutionShape) { + auto verify_strategy = [](bool prefer_indices, + ParquetDictionaryMaterializationStrategy expected_strategy, + size_t expected_direct_batches, size_t expected_index_batches) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + assert_cast(*state.typed_dictionary).get_data() = {10, 20, 30, 40}; + ScriptedDictionaryMaterializationSource source({3, 0, 2, 1, 3}, prefer_indices); + auto output = ColumnInt32::create(); + + const auto status = state.materialize_dictionary(*output, source, 5); + EXPECT_TRUE(status.ok()) << status; + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_materialization_strategy, expected_strategy); + EXPECT_EQ(source.direct_batches, expected_direct_batches); + EXPECT_EQ(source.index_batches, expected_index_batches); + }; + + verify_strategy(false, ParquetDictionaryMaterializationStrategy::DIRECT, 1, 0); + verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (size_t row = 0; row < 64; ++row) { + encoder.Put(1); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 1; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 64, state).ok()); + EXPECT_EQ(state.dictionary_materialization_strategy, + ParquetDictionaryMaterializationStrategy::DIRECT); + ASSERT_EQ(output->size(), 64); + for (size_t row = 0; row < output->size(); ++row) { + EXPECT_EQ(assert_cast(*output).get_element(row), 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DictionaryDirectGatherRollsBackLateCorruptRun) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (size_t row = 0; row < 8; ++row) { + encoder.Put(0); + } + for (size_t row = 0; row < 8; ++row) { + encoder.Put(3); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 2; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + const auto status = + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 16, state); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(output->empty()); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { + const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; + EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); + + auto invalid_tail = valid; + invalid_tail.back() = 8; + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_tail.data(), invalid_tail.size(), 8)); + + auto invalid_unsigned = valid; + invalid_unsigned[3] = std::numeric_limits::max(); + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_unsigned.data(), + invalid_unsigned.size(), 8)); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { + const std::vector dictionary {1, 1000, 2, 3}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeInt8 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(9); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_int32(dictionary, physical_ids, 7, null_runs, + filter, type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {9, 0, 0, 2, 0})); + EXPECT_EQ(null_map, (NullMap {0, 1, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionExpandsStringsInPlace) { + const std::vector dictionary {"a", "bbb", "cc", "dddd"}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeString type; + auto column = type.create_column(); + assert_cast(*column).insert_data("prefix", 6); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_strings(dictionary, physical_ids, 7, null_runs, + filter, &column, &null_map, &statistics) + .ok()); + + ASSERT_EQ(column->size(), 5); + EXPECT_EQ(column->get_data_at(0).to_string_view(), "prefix"); + EXPECT_EQ(column->get_data_at(1).to_string_view(), ""); + EXPECT_EQ(column->get_data_at(2).to_string_view(), "bbb"); + EXPECT_EQ(column->get_data_at(3).to_string_view(), "cc"); + EXPECT_EQ(column->get_data_at(4).to_string_view(), ""); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes) { + DataTypeInt32 type; + ColumnChunkReaderStatistics statistics; + + auto all_null_column = type.create_column(); + assert_cast(*all_null_column).get_data().push_back(8); + NullMap all_null_map; + all_null_map.push_back(0); + const std::vector select_alternating {1, 0, 1, 0, 1}; + ASSERT_TRUE(materialize_selected_plain_int32({}, 5, {0, 5}, select_alternating, type, + &all_null_column, &all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_null_column = type.create_column(); + assert_cast(*dictionary_all_null_column).get_data().push_back(8); + NullMap dictionary_all_null_map; + dictionary_all_null_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10}, {}, 5, {0, 5}, select_alternating, type, + &dictionary_all_null_column, + &dictionary_all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(dictionary_all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto all_filtered_column = type.create_column(); + assert_cast(*all_filtered_column).get_data().push_back(9); + NullMap all_filtered_map; + all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_plain_int32( + {10, 20, 30}, 5, {1, 1, 1, 1, 1}, std::vector(5, 0), type, + &all_filtered_column, &all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_filtered_column = type.create_column(); + assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); + NullMap dictionary_all_filtered_map; + dictionary_all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10, 20, 30}, {0, 1, 2}, 5, {1, 1, 1, 1, 1}, + std::vector(5, 0), type, + &dictionary_all_filtered_column, + &dictionary_all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(dictionary_all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); + const uint8_t* dictionary_address = dictionary.get(); + ByteArrayDictDecoder decoder; + + ASSERT_TRUE(decoder.set_dict(dictionary, dictionary_length, 2).ok()); + EXPECT_EQ(dictionary.get(), nullptr); + + RejectFixedConsumer fixed_consumer; + CaptureBinaryConsumer binary_consumer; + ASSERT_TRUE(decoder.decode_dictionary(fixed_consumer, binary_consumer).ok()); + ASSERT_EQ(binary_consumer.refs.size(), 2); + EXPECT_EQ(binary_consumer.refs[0].to_string_view(), "alpha"); + EXPECT_EQ(binary_consumer.refs[1].to_string_view(), "beta"); + EXPECT_EQ(binary_consumer.refs[0].data, + reinterpret_cast(dictionary_address + sizeof(uint32_t))); + + // bit width 1, an RLE run of three values (header = 3 << 1), dictionary id 1. + char valid_indices[] = {1, 6, 1}; + Slice valid_slice(valid_indices, sizeof(valid_indices)); + ASSERT_TRUE(decoder.set_data(&valid_slice).ok()); + std::vector decoded_indices; + ASSERT_TRUE(decoder.decode_dictionary_indices(3, &decoded_indices).ok()); + EXPECT_EQ(decoded_indices, std::vector({1, 1, 1})); + + // bit width 2, one RLE value with dictionary id 3. Skipping still validates the encoded id so + // filter selection cannot hide a corrupt dictionary stream. + char invalid_indices[] = {2, 2, 3}; + Slice invalid_slice(invalid_indices, sizeof(invalid_indices)); + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetDecodeSource& source = decoder; + EXPECT_TRUE(source.skip_values(1).is()); + + // Sparse decode must validate filtered dictionary ids too. Predicate selection must never + // turn a corrupt page into a successful read merely because the bad row was not selected. + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetSelection filtered_corrupt {.total_values = 1, .selected_values = 0, .ranges = {}}; + EXPECT_TRUE(decoder.decode_selected_dictionary_indices(filtered_corrupt, &decoded_indices) + .is()); + + Slice empty_indices; + EXPECT_TRUE(decoder.set_data(&empty_indices).is()); + + // Dictionary indices are uint32_t. Wider external bit widths would make the RLE decoder copy + // a five-byte repeated value into four-byte state before it can validate an index. + char oversized_bit_width[] = {33, 2, 0}; + Slice oversized_bit_width_slice(oversized_bit_width, sizeof(oversized_bit_width)); + EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryBoundsEntryCountBeforeReserve) { + auto dictionary = make_unique_buffer(1); + dictionary.get()[0] = 0; + ByteArrayDictDecoder decoder; + EXPECT_TRUE(decoder.set_dict(dictionary, 1, std::numeric_limits::max()) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { + TimezoneUtils::load_timezones_to_cache(); + for (const auto converted_type : + {tparquet::ConvertedType::TIMESTAMP_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS}) { + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT64; + field.parquet_schema.__set_converted_type(converted_type); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.logical_type, ParquetLogicalType::TIMESTAMP); + EXPECT_TRUE(context.timestamp_is_adjusted_to_utc); + + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + ASSERT_TRUE(init_decode_context_for_test(field, &shanghai, &context).ok()); + int64_t epoch = 0; + Slice epoch_slice(reinterpret_cast(&epoch), sizeof(epoch)); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(sizeof(epoch)); + ASSERT_TRUE(decoder->set_data(&epoch_slice).ok()); + ParquetMaterializationState state; + DataTypeDateTimeV2 type(0); + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, *decoder, context, 1, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 08:00:00"); + } +} + +TEST(ParquetV2NativeDecoderTest, GeospatialByteArrayAnnotationsDecodeAsRawBytes) { + const std::string wkb("\x01\x01\x00\x00\x00", 5); + for (const bool geometry : {true, false}) { + tparquet::LogicalType logical; + if (geometry) { + logical.__set_GEOMETRY(tparquet::GeometryType()); + } else { + logical.__set_GEOGRAPHY(tparquet::GeographyType()); + } + + NativeFieldSchema field; + field.name = geometry ? "geometry" : "geography"; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.physical_type, ParquetPhysicalType::BYTE_ARRAY); + EXPECT_EQ(context.logical_type, ParquetLogicalType::NONE); + + DataTypeString type; + auto plain_column = type.create_column(); + auto encoded = encode_plain_byte_arrays({wkb}); + Slice plain_slice(encoded.data(), encoded.size()); + std::unique_ptr plain_decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, + plain_decoder) + .ok()); + ASSERT_TRUE(plain_decoder->set_data(&plain_slice).ok()); + ParquetMaterializationState plain_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*plain_column, *plain_decoder, context, 1, + plain_state) + .ok()); + ASSERT_EQ(plain_column->size(), 1); + EXPECT_EQ(plain_column->get_data_at(0).to_string_view(), wkb); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({wkb, "unused"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 2).ok()); + char dictionary_index[] = {1, 2, 0}; + Slice dictionary_slice(dictionary_index, sizeof(dictionary_index)); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + context.encoding = ParquetValueEncoding::DICTIONARY; + auto dictionary_column = type.create_column(); + ParquetMaterializationState dictionary_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, dictionary_decoder, + context, 1, dictionary_state) + .ok()); + ASSERT_EQ(dictionary_column->size(), 1); + EXPECT_EQ(dictionary_column->get_data_at(0).to_string_view(), wkb); + + field.physical_type = tparquet::Type::INT32; + EXPECT_TRUE( + init_decode_context_for_test(field, nullptr, &context).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { + auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + + tparquet::LogicalType date; + date.__set_DATE(tparquet::DateType()); + EXPECT_FALSE(invalid(tparquet::Type::BOOLEAN, date).ok()); + + tparquet::LogicalType timestamp; + timestamp.__set_TIMESTAMP(tparquet::TimestampType()); + timestamp.TIMESTAMP.__set_unit(tparquet::TimeUnit()); + timestamp.TIMESTAMP.unit.__set_MICROS(tparquet::MicroSeconds()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, timestamp).ok()); + + tparquet::LogicalType integer; + integer.__set_INTEGER(tparquet::IntType()); + integer.INTEGER.__set_bitWidth(64); + integer.INTEGER.__set_isSigned(true); + EXPECT_FALSE(invalid(tparquet::Type::INT32, integer).ok()); + + tparquet::LogicalType string; + string.__set_STRING(tparquet::StringType()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, string).ok()); + + tparquet::LogicalType uuid; + uuid.__set_UUID(tparquet::UUIDType()); + EXPECT_FALSE(invalid(tparquet::Type::BYTE_ARRAY, uuid).ok()); + + auto invalid_converted = [](tparquet::Type::type physical, + tparquet::ConvertedType::type converted, int type_length = -1, + int precision = -1, int scale = -1) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_converted_type(converted); + if (type_length >= 0) { + field.parquet_schema.__set_type_length(type_length); + } + if (precision >= 0) { + field.parquet_schema.__set_precision(precision); + } + if (scale >= 0) { + field.parquet_schema.__set_scale(scale); + } + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UTF8).ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::TIME_MICROS).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UINT_64).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::ConvertedType::INTERVAL, 8) + .ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::DECIMAL, -1, 10, 2) + .ok()); + + NativeFieldSchema fixed; + fixed.name = "fixed"; + fixed.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + ParquetDecodeContext context; + EXPECT_FALSE(init_decode_context_for_test(fixed, nullptr, &context).ok()); + fixed.parquet_schema.__set_type_length(4); + EXPECT_TRUE(init_decode_context_for_test(fixed, nullptr, &context).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema int96_field; + int96_field.physical_type = tparquet::Type::INT96; + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, false)); + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, true)); + + NativeFieldSchema utc_field; + utc_field.physical_type = tparquet::Type::INT64; + utc_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + utc_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + utc_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(utc_field, datetime_type, false)); + + NativeFieldSchema converted_utc_field; + converted_utc_field.physical_type = tparquet::Type::INT64; + converted_utc_field.parquet_schema.__set_converted_type( + tparquet::ConvertedType::TIMESTAMP_MICROS); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(converted_utc_field, datetime_type, + false)); + + NativeFieldSchema local_field = utc_field; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + EXPECT_FALSE( + preserves_timestamp_conversion_default_for_test(local_field, datetime_type, false)); + + NativeFieldSchema integer_field; + integer_field.physical_type = tparquet::Type::INT64; + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test( + integer_field, make_nullable(std::make_shared()), false)); +} + +TEST(ParquetV2NativeDecoderTest, FastInt96NormalizesLegacyOutOfDayNanos) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema local_field; + local_field.physical_type = tparquet::Type::INT64; + local_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + local_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + const uint64_t invalid_datetime = 0; + column->insert_data(reinterpret_cast(&invalid_datetime), sizeof(invalid_datetime)); + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 1); + + null_map[0] = 0; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { + const ParquetSelection selection { + .total_values = 7, + .selected_values = 3, + .ranges = {{.first = 1, .count = 2}, {.first = 6, .count = 1}}}; + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {10, 11, 12, 13, 14, 15, 16, 17}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({11, 12, 16})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({17})); + + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded_strings = encode_plain_byte_arrays(strings); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + Slice string_slice(encoded_strings.data(), encoded_strings.size()); + ASSERT_TRUE(decoder->set_data(&string_slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 3); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "one"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "two"); + EXPECT_EQ(selected_strings.refs[2].to_string_view(), "six"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "seven"); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char booleans[] = {static_cast(0b10001101)}; + Slice boolean_slice(booleans, sizeof(booleans)); + ASSERT_TRUE(decoder->set_data(&boolean_slice).ok()); + CaptureFixedConsumer selected_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_booleans).ok()); + EXPECT_EQ(selected_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_boolean).ok()); + EXPECT_EQ(trailing_boolean.values(), std::vector({1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_booleans[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_booleans, sizeof(rle_booleans)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + CaptureFixedConsumer selected_rle_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_rle_booleans).ok()); + EXPECT_EQ(selected_rle_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_rle_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_rle_boolean).ok()); + EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); +} + +TEST(ParquetV2NativeDecoderTest, PlainByteArrayPublishesOffsetsAndCoalescedSelectionSpans) { + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded = encode_plain_byte_arrays(strings); + Slice slice(encoded.data(), encoded.size()); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + + const ParquetSelection selection { + .total_values = 7, + .selected_values = 4, + .ranges = {{.first = 1, .count = 2}, {.first = 5, .count = 2}}}; + CapturePlainBinaryLayoutConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, consumer).ok()); + EXPECT_FALSE(consumer.legacy_consume_called); + EXPECT_EQ(consumer.output_offsets, std::vector({0, 3, 6, 10, 13})); + ASSERT_EQ(consumer.source_offsets.size(), selection.selected_values); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[0], 3), "one"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[1], 3), "two"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[2], 4), "five"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[3], 3), "six"); + ASSERT_EQ(consumer.spans.size(), 2); + EXPECT_EQ(consumer.spans[0].first, 0); + EXPECT_EQ(consumer.spans[0].count, 2); + EXPECT_EQ(consumer.spans[1].first, 2); + EXPECT_EQ(consumer.spans[1].count, 2); + + CaptureBinaryConsumer trailing; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing).ok()); + ASSERT_EQ(trailing.refs.size(), 1); + EXPECT_EQ(trailing.refs[0].to_string_view(), "seven"); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { + constexpr size_t value_count = 1UL << 18; + std::vector integers(value_count); + std::iota(integers.begin(), integers.end(), 0); + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 2, + .ranges = {{.first = 1, .count = 1}, {.first = value_count - 1, .count = 1}}}; + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), + std::vector({1, static_cast(value_count - 1)})); + // Sparse PLAIN spans can be consumed directly from the encoded page. Retaining a gather buffer + // makes a highly selective batch allocate in proportion to its selected width for no benefit. + EXPECT_EQ(decoder->retained_scratch_bytes(), 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedPlainLargeSkipCannotWrapPageOffset) { + std::vector values {11, 22}; + Slice slice(reinterpret_cast(values.data()), values.size() * sizeof(int64_t)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int64_t)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(536870913).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({11})); +} + +TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {11, 22, 33}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer integer_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, integer_consumer).ok()); + EXPECT_EQ(integer_consumer.values(), std::vector({22, 33})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0b10001101)}; + Slice plain_boolean_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_boolean_slice).ok()); + CaptureFixedConsumer plain_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(8, plain_boolean_consumer).ok()); + EXPECT_EQ(plain_boolean_consumer.values(), + std::vector({1, 0, 1, 1, 0, 0, 0, 1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_boolean[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_boolean, sizeof(rle_boolean)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + ASSERT_TRUE(decoder->skip_values(3).ok()); + CaptureFixedConsumer rle_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(5, rle_boolean_consumer).ok()); + EXPECT_EQ(rle_boolean_consumer.values(), std::vector({1, 0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer first_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, first_integer).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer remaining_integers; + ASSERT_TRUE(decoder->decode_fixed_values(2, remaining_integers).ok()); + EXPECT_EQ(first_integer.values(), std::vector({100})); + EXPECT_EQ(remaining_integers.values(), std::vector({99, 1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(3, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 3); + EXPECT_EQ(consumer.refs[0].to_string_view(), "prefix-b"); + EXPECT_EQ(consumer.refs[1].to_string_view(), "other"); + EXPECT_EQ(consumer.refs[2].to_string_view(), "other-tail"); + } +} + +TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { + const ParquetSelection selection { + .total_values = 3, + .selected_values = 2, + .ranges = {{.first = 0, .count = 1}, {.first = 2, .count = 1}}}; + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({100, 99})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 2); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "prefix-a"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "other"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "other-tail"); + } + + const std::vector floats {1.0F, -2.5F, 3.25F, 9.5F}; + std::vector encoded_floats(floats.size() * sizeof(float)); + for (size_t row = 0; row < floats.size(); ++row) { + const auto* bytes = reinterpret_cast(&floats[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded_floats[byte * floats.size() + row] = bytes[byte]; + } + } + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice float_slice(encoded_floats.data(), encoded_floats.size()); + ASSERT_TRUE(decoder->set_data(&float_slice).ok()); + CaptureFixedConsumer selected_floats; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_floats).ok()); + EXPECT_EQ(selected_floats.values(), std::vector({1.0F, 3.25F})); + CaptureFixedConsumer trailing_float; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_float).ok()); + EXPECT_EQ(trailing_float.values(), std::vector({9.5F})); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { + const std::vector values {1.0F, -2.5F, 3.25F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t row = 0; row < values.size(); ++row) { + const auto* bytes = reinterpret_cast(&values[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded[byte * values.size() + row] = bytes[byte]; + } + } + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) { + char encoded[] = {static_cast(0b00001101)}; + Slice levels(encoded, sizeof(encoded)); + LevelDecoder decoder; + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 1, 4).ok()); + + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 4), 1); + EXPECT_EQ(value, 1); + EXPECT_EQ(decoder.get_next(), 0); + decoder.rewind_one(); + EXPECT_EQ(decoder.get_next(), 0); + level_t tail[2] = {-1, -1}; + EXPECT_EQ(decoder.get_levels(tail, 2), 2); + EXPECT_EQ(tail[0], 1); + EXPECT_EQ(tail[1], 1); + + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice truncated_levels(truncated_rle, sizeof(truncated_rle)); + LevelDecoder rle_decoder; + ASSERT_TRUE(rle_decoder.init(&truncated_levels, tparquet::Encoding::RLE, 1, 1).ok()); + level_t truncated_value = -1; + EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelByteCountDoesNotWrapAtLargeCounts) { + char placeholder[8] = {}; + constexpr uint32_t num_levels = 1'500'000'000; + constexpr size_t expected_bytes = 562'500'000; + Slice levels(placeholder, expected_bytes); + LevelDecoder decoder; + + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 4, num_levels).ok()); + EXPECT_EQ(levels.size, 0); +} + +TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { + auto make_decoder = [](tparquet::Encoding::type encoding) { + static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. + static char rle_encoded[] = {0x02, 0x00, 0x00, 0x00, 0x02, 0x03}; + Slice levels = encoding == tparquet::Encoding::BIT_PACKED + ? Slice(encoded, sizeof(encoded)) + : Slice(rle_encoded, sizeof(rle_encoded)); + LevelDecoder decoder; + EXPECT_TRUE(decoder.init(&levels, encoding, 2, 1).ok()); + return decoder; + }; + + for (auto encoding : {tparquet::Encoding::BIT_PACKED, tparquet::Encoding::RLE}) { + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_levels(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + EXPECT_EQ(decoder.get_next(), -1); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedReadersRejectRleAndBitPackedLevelsAboveMaximum) { + for (auto encoding : {tparquet::Encoding::RLE, tparquet::Encoding::BIT_PACKED}) { + const std::vector payload = encoding == tparquet::Encoding::RLE + ? std::vector {2, 0, 0, 0, 2, 3} + : std::vector {3}; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + + EXPECT_TRUE(load_malformed_nested_page(header, payload, 1, 2).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0xFF)}; + Slice plain_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_slice).ok()); + EXPECT_FALSE(decoder->skip_values(9).ok()); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + // A bit-packed run header for eight values without its required payload byte. + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice rle_slice(truncated_rle, sizeof(truncated_rle)); + ASSERT_TRUE(decoder->set_data(&rle_slice).ok()); + EXPECT_FALSE(decoder->skip_values(1).ok()); + + // The first literal group is complete but the second has only its header. Exact-count checks + // must reject both dense and selected decodes instead of exposing the retained vector tail. + char partially_truncated_rle[] = {0x03, 0x00, 0x00, 0x00, 0x03, static_cast(0xFF), 0x03}; + Slice partially_truncated_slice(partially_truncated_rle, sizeof(partially_truncated_rle)); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer dense_consumer; + EXPECT_FALSE(decoder->decode_fixed_values(9, dense_consumer).ok()); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_tail { + .total_values = 9, .selected_values = 1, .ranges = {{.first = 8, .count = 1}}}; + EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); +} + +TEST(ParquetV2NativeDecoderTest, CompactRleSkipsKeepScratchBounded) { + constexpr uint32_t value_count = 1U << 20; + uint8_t header[16]; + uint8_t* header_end = encode_varint32(header, value_count << 1); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"only"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 1).ok()); + std::vector dictionary_indices {0}; + dictionary_indices.insert(dictionary_indices.end(), reinterpret_cast(header), + reinterpret_cast(header_end)); + Slice dictionary_slice(dictionary_indices.data(), dictionary_indices.size()); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + ParquetDecodeSource& dictionary_source = dictionary_decoder; + ASSERT_TRUE(dictionary_source.skip_values(value_count).ok()); + EXPECT_LE(dictionary_decoder.retained_scratch_bytes(), 4096 * sizeof(uint32_t)); + + std::vector boolean_payload(reinterpret_cast(header), + reinterpret_cast(header_end)); + boolean_payload.push_back(0); + std::vector boolean_page(sizeof(uint32_t) + boolean_payload.size()); + encode_fixed32_le(reinterpret_cast(boolean_page.data()), boolean_payload.size()); + memcpy(boolean_page.data() + sizeof(uint32_t), boolean_payload.data(), boolean_payload.size()); + Slice boolean_slice(boolean_page.data(), boolean_page.size()); + std::unique_ptr boolean_decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, boolean_decoder) + .ok()); + ASSERT_TRUE(boolean_decoder->set_data(&boolean_slice).ok()); + ASSERT_TRUE(boolean_decoder->skip_values(value_count).ok()); + EXPECT_LE(boolean_decoder->retained_scratch_bytes(), 4096); +} + +TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { + constexpr uint32_t delta_count = 1U << 20; + std::vector encoded(64); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, delta_count); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, delta_count + 1); + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta + *cursor++ = 0; // zero-width miniblock + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(delta_count + 1); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(delta_count + 1).ok()); + EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); +} + +TEST(ParquetV2NativeDecoderTest, DeltaPaddingBitCountIsWidenedBeforeCursorAdvance) { + int64_t padding_bits = 0; + ASSERT_TRUE(detail::checked_delta_padding_bits(64, std::numeric_limits::max(), + &padding_bits) + .ok()); + EXPECT_EQ(padding_bits, 64LL * std::numeric_limits::max()); + EXPECT_GT(padding_bits, std::numeric_limits::max()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { + constexpr size_t value_count = 1U << 16; + std::vector<::parquet::ByteArray> values(value_count); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + static_cast(encoding), decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(value_count).ok()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayLongPrefixSparseWorkIsByteBounded) { + constexpr size_t value_count = 128; + const std::string repeated_value(256UL << 10, 'x'); + std::vector<::parquet::ByteArray> values( + value_count, + ::parquet::ByteArray(static_cast(repeated_value.size()), + reinterpret_cast(repeated_value.data()))); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + auto make_decoder = [&]() { + std::unique_ptr decoder; + EXPECT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + EXPECT_TRUE(decoder->set_data(&slice).ok()); + return decoder; + }; + + auto skipped = make_decoder(); + ASSERT_TRUE(skipped->skip_values(value_count).ok()); + EXPECT_LT(skipped->retained_scratch_bytes(), 8UL << 20); + + auto sparse = make_decoder(); + CaptureBinaryConsumer consumer; + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 1, + .ranges = {{.first = value_count - 1, .count = 1}}, + }; + ASSERT_TRUE(sparse->decode_selected_binary_values(selection, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 1); + EXPECT_EQ(consumer.refs[0].to_string_view(), repeated_value); + EXPECT_LT(sparse->retained_scratch_bytes(), 8UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { + const std::vector values {"good", "bad"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_type_length(4); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_good { + .total_values = 2, .selected_values = 1, .ranges = {{.first = 0, .count = 1}}}; + EXPECT_TRUE(decoder->decode_selected_fixed_values(select_good, selected_consumer) + .is()); + + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_TRUE(decoder->skip_values(2).is()); +} + +TEST(ParquetV2NativeDecoderTest, TruncatedFixedWidthPlainKeepsPublicErrorKeyword) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(4); + char truncated[] = {'a', 'b', 'c'}; + Slice slice(truncated, sizeof(truncated)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + const auto status = decoder->decode_fixed_values(1, consumer); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unexpected end of stream"), std::string::npos); +} + +TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { + const std::vector integers {7}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(integers.data(), static_cast(integers.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(2).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocation) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + // block_size=128, miniblocks=4, total_values=INT_MAX, first_value=0. + char oversized_count[] = {static_cast(0x80), + 0x01, + 0x04, + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + 0x07, + 0x00}; + Slice oversized_count_slice(oversized_count, sizeof(oversized_count)); + EXPECT_TRUE(decoder->set_data(&oversized_count_slice).is()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + // A single decoded length of 1 GiB with no following payload bytes must fail before the + // decoder resizes its backing data buffer. + char oversized_length[] = {static_cast(0x80), + 0x01, + 0x04, + 0x01, + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + 0x08}; + Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); + CaptureBinaryConsumer consumer; + EXPECT_TRUE(decoder->set_data(&oversized_length_slice).is()); + EXPECT_TRUE(consumer.refs.empty()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + std::vector huge_prefix(64); + uint8_t* cursor = huge_prefix.data(); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, std::numeric_limits::max() - 1); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); // one empty suffix + huge_prefix.resize(cursor - huge_prefix.data()); + Slice huge_prefix_slice(huge_prefix.data(), huge_prefix.size()); + ASSERT_TRUE(decoder->set_data(&huge_prefix_slice).ok()); + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { + const std::vector strings {"old", "state"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + Slice valid(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&valid).ok()); + Slice empty; + EXPECT_TRUE(decoder->set_data(&empty).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + char partial_row[] = {0, 1, 2, 3, 4}; + Slice slice(partial_row, sizeof(partial_row)); + EXPECT_TRUE(decoder->set_data(&slice).is()); +} + +Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, + size_t key_values, size_t value_values, + std::vector key_rep_levels, std::vector value_rep_levels, + bool value_eof, std::vector key_def_levels = {}, + std::vector value_def_levels = {}) { + auto value_type = make_nullable(std::make_shared()); + auto map_type = std::make_shared(key_type, value_type); + ColumnPtr column = map_type->create_column(); + NativeFieldSchema field; + field.name = "m"; + field.data_type = map_type; + field.definition_level = 1; + field.repetition_level = 1; + field.repeated_parent_def_level = 0; + + if (key_def_levels.empty()) { + key_def_levels.assign(key_rep_levels.size(), 1); + } + if (value_def_levels.empty()) { + value_def_levels.assign(value_rep_levels.size(), 1); + } + auto key_reader = std::make_unique( + key_rows, key_values, true, std::move(key_rep_levels), std::move(key_def_levels)); + auto value_reader = std::make_unique(value_rows, value_values, value_eof, + std::move(value_rep_levels), + std::move(value_def_levels)); + MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); + RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); + + auto root = std::make_shared(std::make_shared(), + std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + return reader.read_column_data(column, map_type, root, filter, key_rows, &read_rows, &eof, + false); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 2, 1}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryDistribution) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 4, 4, {0, 1, 0, 1}, {0, 1, 1, 0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryPresence) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 2, 2, {0, 0}, {0, 0}, true, {0, 1}, {1, 0}) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { + auto int_type = std::make_shared(); + EXPECT_TRUE( + read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderPreservesNullableKeysWrittenByDoris) { + auto nullable_key = make_nullable(std::make_shared()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 0}); + children["b"] = std::make_unique(1, 1, true, std::vector {0}, + std::vector {0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.repetition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique( + 2, 3, true, std::vector {0, 1, 0}, std::vector {0, 0, 0}); + children["b"] = std::make_unique( + 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedOptionalParentPresence) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.definition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 1}); + children["b"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {1, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { + constexpr size_t value_count = 1UL << 20; + std::vector encoded(value_count * sizeof(float), 0); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + EXPECT_EQ(decoder->active_scratch_bytes(), value_count * sizeof(float)); + + std::vector ordinary_encoded(sizeof(float), 0); + Slice ordinary_slice(ordinary_encoded.data(), ordinary_encoded.size()); + ASSERT_TRUE(decoder->set_data(&ordinary_slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + // Capacity records the high-water allocation while size records this batch. The reader needs + // both to distinguish stable large batches from a one-off outlier before releasing capacity. + EXPECT_EQ(decoder->active_scratch_bytes(), sizeof(float)); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + decoder->release_scratch(64UL << 10); + EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { + const std::vector values {std::string(4096, 'x'), "shared-prefix-a", + "shared-prefix-b", "shared-prefix-c"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + decoder->release_scratch(64); + ASSERT_TRUE(decoder->decode_binary_values(2, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 2); + EXPECT_EQ(consumer.refs[0].to_string_view(), values[2]); + EXPECT_EQ(consumer.refs[1].to_string_view(), values[3]); +} + +TEST(ParquetV2NativeDecoderTest, DeltaBitPackScratchReleasePreservesMiniblockState) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + + std::vector outlier_header(64); + uint8_t* cursor = outlier_header.data(); + cursor = encode_varint32(cursor, 512); // values per block + cursor = encode_varint32(cursor, 16); // miniblocks per block + cursor = encode_varint32(cursor, 2); // total values + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta, zig-zag encoded + memset(cursor, 0, 16); // one zero bit width per miniblock + cursor += 16; + outlier_header.resize(cursor - outlier_header.data()); + decoder->set_expected_values(2); + Slice outlier_slice(outlier_header.data(), outlier_header.size()); + ASSERT_TRUE(decoder->set_data(&outlier_slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + decoder->release_scratch(8); + ASSERT_TRUE(decoder->decode_fixed_values(values.size() - 1, consumer).ok()); + EXPECT_EQ(consumer.values().size(), values.size() + 1); +} + +TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { + auto parse_header = [](tparquet::PageHeader header) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + if (header.compressed_page_size > 0) { + bytes.resize(bytes.size() + header.compressed_page_size); + } + MemoryBufferedReader reader(bytes); + tparquet::ColumnMetaData metadata; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, bytes.size(), 1, metadata, + context); + return page_reader.parse_page_header(); + }; + + tparquet::PageHeader negative; + negative.type = tparquet::PageType::DATA_PAGE; + negative.__set_compressed_page_size(-1); + negative.__set_uncompressed_page_size(1); + negative.__isset.data_page_header = true; + negative.data_page_header.__set_num_values(1); + EXPECT_TRUE(parse_header(negative).is()); + + tparquet::PageHeader oversized_levels; + oversized_levels.type = tparquet::PageType::DATA_PAGE_V2; + oversized_levels.__set_compressed_page_size(1); + oversized_levels.__set_uncompressed_page_size(1); + oversized_levels.__isset.data_page_header_v2 = true; + oversized_levels.data_page_header_v2.__set_num_values(1); + oversized_levels.data_page_header_v2.__set_num_rows(1); + oversized_levels.data_page_header_v2.__set_repetition_levels_byte_length(2); + oversized_levels.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(oversized_levels).is()); + + tparquet::PageHeader impossible_counts = oversized_levels; + impossible_counts.__set_compressed_page_size(0); + impossible_counts.__set_uncompressed_page_size(0); + impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); + impossible_counts.data_page_header_v2.__set_num_nulls(2); + EXPECT_TRUE(parse_header(impossible_counts).is()); + + tparquet::PageHeader missing_layout; + missing_layout.type = tparquet::PageType::DATA_PAGE; + missing_layout.__set_compressed_page_size(0); + missing_layout.__set_uncompressed_page_size(0); + EXPECT_TRUE(parse_header(missing_layout).is()); + + auto swapped_layout = oversized_levels; + swapped_layout.type = tparquet::PageType::DATA_PAGE; + EXPECT_TRUE(parse_header(swapped_layout).is()); + + auto competing_layouts = negative; + competing_layouts.__set_compressed_page_size(0); + competing_layouts.__set_uncompressed_page_size(0); + competing_layouts.__isset.data_page_header_v2 = true; + competing_layouts.data_page_header_v2.__set_num_values(0); + competing_layouts.data_page_header_v2.__set_num_rows(0); + competing_layouts.data_page_header_v2.__set_num_nulls(0); + competing_layouts.data_page_header_v2.__set_repetition_levels_byte_length(0); + competing_layouts.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(competing_layouts).is()); +} + +TEST(ParquetV2NativeDecoderTest, ShiftedOffsetIndexFallsBackToSequentialPages) { + auto data_page = [](int32_t value) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(sizeof(value)); + header.__set_uncompressed_page_size(sizeof(value)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(sizeof(value)); + memcpy(payload.data(), &value, sizeof(value)); + return serialize_page(header, payload); + }; + auto verify_fallback = [&](bool cache_hit) { + const auto first_page = data_page(10); + const auto second_page = data_page(20); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + bytes.push_back(0); + + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(cast_set(first_page.size() + 1)); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(cast_set(first_page.size() + 1)); + second_location.__set_compressed_page_size(cast_set(second_page.size())); + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + EXPECT_TRUE( + validate_offset_index(offset_index, {.offset = 0, .length = bytes.size()}, 0, 2)); + + const std::string cache_key = cache_hit ? "shifted-index-cache-hit" : ""; + if (cache_hit) { + auto* page = new DataPage(first_page.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), first_page.data(), first_page.size()); + page->reset_size(first_page.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(cache_key, bytes.size(), 0), page, &handle, + segment_v2::DATA_PAGE); + } + + tparquet::ColumnMetaData metadata; + metadata.__set_type(tparquet::Type::INT32); + metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + metadata.__set_num_values(2); + metadata.__set_total_compressed_size(bytes.size()); + metadata.__set_data_page_offset(0); + MemoryBufferedReader stream(std::move(bytes)); + PageReader reader(&stream, nullptr, 0, metadata.total_compressed_size, 2, + metadata, ParquetPageReadContext(cache_hit, cache_key), + &offset_index); + + ASSERT_TRUE(reader.parse_page_header().ok()); + reader.skip_page_data(); + ASSERT_TRUE(reader.next_page().ok()); + // The rectangles above are non-overlapping but shifted one byte from the serialized pages. + // Discarding the optional index must continue from the first page's real payload end. + ASSERT_TRUE(reader.parse_page_header().ok()); + const tparquet::PageHeader* parsed = nullptr; + ASSERT_TRUE(reader.get_page_header(&parsed).ok()); + EXPECT_EQ(parsed->data_page_header.num_values, 1); + }; + verify_fallback(false); + verify_fallback(true); +} + +TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { + auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) { + std::vector payload(static_cast(header.compressed_page_size), 0); + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + if (with_offset_index) { + ColumnChunkReader reader_with_index(&reader, &chunk, &field, &offset_index, + 1, nullptr, context); + return reader_with_index.init(); + } + ColumnChunkReader sequential_reader(&reader, &chunk, &field, nullptr, 1, + nullptr, context); + return sequential_reader.init(); + }; + + tparquet::PageHeader v2; + v2.type = tparquet::PageType::DATA_PAGE_V2; + v2.__set_compressed_page_size(8); + v2.__set_uncompressed_page_size(8); + v2.__isset.data_page_header_v2 = true; + v2.data_page_header_v2.__set_num_values(2); + v2.data_page_header_v2.__set_num_rows(1); + v2.data_page_header_v2.__set_num_nulls(0); + v2.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + v2.data_page_header_v2.__set_repetition_levels_byte_length(0); + v2.data_page_header_v2.__set_definition_levels_byte_length(0); + v2.data_page_header_v2.__set_is_compressed(false); + auto status = init_chunk(v2, false); + EXPECT_TRUE(status.is()) << status; + status = init_chunk(v2, true); + EXPECT_TRUE(status.is()) << status; + + tparquet::PageHeader v1; + v1.type = tparquet::PageType::DATA_PAGE; + v1.__set_compressed_page_size(8); + v1.__set_uncompressed_page_size(8); + v1.__isset.data_page_header = true; + v1.data_page_header.__set_num_values(2); + v1.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + v1.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + v1.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + status = init_chunk(v1, true); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsOffsetIndexRowSpanMismatch) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(8); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + auto bytes = serialize_page(header, std::vector(8)); + MemoryBufferedReader stream(bytes); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ParquetPageReadContext context(false, ""); + + ColumnChunkReader reader(&stream, &chunk, &field, &offset_index, + /*total_rows=*/2, nullptr, context); + const auto status = reader.init(); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyAllNullPagesInitializeZeroValueDecoders) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, true); + EXPECT_TRUE(status.ok()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyPagesRejectDefinitionLevelsThatRequireValues) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, false); + EXPECT_TRUE(status.is()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyReaderSkipsLeadingZeroValuePages) { + auto page = [](bool v2, std::optional value) { + std::vector payload; + if (value.has_value()) { + const std::vector definition_levels {2, 1}; + if (v2) { + payload = definition_levels; + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), definition_levels.size()); + payload.insert(payload.end(), definition_levels.begin(), definition_levels.end()); + } + const auto* value_bytes = reinterpret_cast(&*value); + payload.insert(payload.end(), value_bytes, value_bytes + sizeof(*value)); + } else if (!v2) { + // V1 prefixes its RLE definition-level stream even when that stream has no values. + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), 0); + } + tparquet::PageHeader header; + header.type = v2 ? tparquet::PageType::DATA_PAGE_V2 : tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + if (v2) { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_rows(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_definition_levels_byte_length(value.has_value() ? 2 + : 0); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + return serialize_page(header, payload); + }; + + static const auto utc = cctz::utc_time_zone(); + for (const bool v2 : {false, true}) { + SCOPED_TRACE(v2 ? "v2" : "v1"); + auto bytes = page(v2, std::nullopt); + const auto nonempty_page = page(v2, 7); + bytes.insert(bytes.end(), nonempty_page.begin(), nonempty_page.end()); + auto file = std::make_shared(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = make_nullable(std::make_shared()); + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + auto row_ranges = ::doris::RowRanges::create_single(1); + ScalarColumnReader scalar_reader(row_ranges, 1, chunk, nullptr, &utc, + nullptr); + ASSERT_TRUE( + scalar_reader + .init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 1, false).ok()); + ColumnPtr scalar_column = field.data_type->create_column(); + size_t scalar_rows = 0; + bool scalar_eof = false; + while (scalar_rows == 0 && !scalar_eof) { + size_t read_now = 0; + ASSERT_TRUE(scalar_reader + .read_column_data(scalar_column, field.data_type, nullptr, filter, + 1, &read_now, &scalar_eof, false) + .ok()); + scalar_rows += read_now; + } + ASSERT_EQ(scalar_rows, 1); + const auto& scalar_nullable = assert_cast(*scalar_column); + EXPECT_FALSE(scalar_nullable.is_null_at(0)); + EXPECT_EQ( + assert_cast(scalar_nullable.get_nested_column()).get_element(0), + 7); + + std::unique_ptr reader; + ASSERT_TRUE(LevelReader::create(file, chunk, &field, 1, bytes.size(), nullptr, false, "", + ParquetReaderCompat {}, &reader) + .ok()); + + std::vector repetition_levels; + std::vector definition_levels; + size_t rows_read = 0; + const auto status = + reader->read_rows(1, &repetition_levels, &definition_levels, &rows_read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows_read, 1); + EXPECT_EQ(repetition_levels, (std::vector {0})); + EXPECT_EQ(definition_levels, (std::vector {1})); + } +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(12); + header.__set_uncompressed_page_size(12); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(2); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(4); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + // Two values [0, 1] contain only one repetition-level zero, so they describe one row. + EXPECT_TRUE(load_malformed_nested_page(header, {2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 2) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, FirstNestedV1PageRejectsOrphanContinuation) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + // A two-value RLE run at repetition level 1 has no level-0 row start. + const std::vector payload {2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(2); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + EXPECT_TRUE(load_malformed_nested_page(header, payload, 2).is()); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1ContinuationRemainsValidAfterFirstRowStart) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + first_page.insert(first_page.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader reader(first_page); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(first_page.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1IgnoresUnverifiableOffsetIndexRows) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + const auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + const auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(first_page.size()); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(first_page.size()); + second_location.__set_compressed_page_size(second_page.size()); + // This V1 page continues row zero, but its index claims a new logical row. Repetition levels + // are the only authoritative source, so indexed seeking must be disabled for this chunk. + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1DiscardedOffsetIndexStopsAtLogicalChunkEnd) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + const std::vector payload {2, 0, 0, 0, 2, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page(header, payload); + const size_t page_size = bytes.size(); + // A compatibility reader may pad the validated physical range beyond the encoded chunk. + bytes.resize(page_size + 16, 0); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(page_size); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(page_size); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ColumnChunkRange padded_range {.offset = 0, .length = bytes.size()}; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context, &padded_range); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + EXPECT_EQ(rows, 1); + EXPECT_FALSE(cross_page); + EXPECT_FALSE(chunk_reader.has_next_page()); +} + +TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { + for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(std::numeric_limits::max()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(std::numeric_limits::max()); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } + // The four-byte V1 level length (or V2 value payload) contains no advertised levels. The + // reader must report corruption without reserving INT_MAX level slots first. + EXPECT_TRUE(load_malformed_nested_page(header, {0, 0, 0, 0}).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + std::vector input(8, 7); + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(input.data(), input.size()), &compressed).ok()); + std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + for (const int32_t advertised_size : {4, 12}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(advertised_size); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + EXPECT_FALSE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndLogicalSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(1); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(load_scripted_page(header, std::vector(8), + tparquet::CompressionCodec::UNCOMPRESSED) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { + for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + } + const std::vector payload(8); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED) + .is()); + EXPECT_TRUE( + load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED, true) + .is()); + } +} + +TEST(ParquetV2NativeDecoderTest, LegacyV2CompressedOverrideAllowsDifferentSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(16); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, true) + .ok()); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { + MemoryBufferedReader reader(std::vector {0}); + tparquet::ColumnMetaData metadata; + tparquet::OffsetIndex empty_index; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, 1, 1, metadata, context, &empty_index); + EXPECT_FALSE(page_reader.has_next_page()); + EXPECT_TRUE(page_reader.next_page().is()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsLegacyPadding) { + tparquet::ColumnMetaData metadata; + metadata.__set_data_page_offset(-1); + metadata.__set_total_compressed_size(10); + ColumnChunkRange range; + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(95); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(10); + metadata.__set_total_compressed_size(std::numeric_limits::max()); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_total_compressed_size(20); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 25); + + metadata.__set_dictionary_page_offset(0); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 20); + + metadata.__set_dictionary_page_offset(5); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 5); + EXPECT_EQ(range.length, 20); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(110); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(120); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_offset(140); + second.__set_first_row_index(0); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_TRUE(validate_offset_index(index, range, 110, 20)); + + range = {.offset = std::numeric_limits::max(), .length = 2}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsShiftedFirstDataPage) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(120); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(140); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + + EXPECT_FALSE(validate_offset_index(index, range, 100, 20)); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { + tparquet::PageHeader index_header; + index_header.type = tparquet::PageType::INDEX_PAGE; + index_header.__set_compressed_page_size(3); + index_header.__set_uncompressed_page_size(3); + index_header.__set_index_page_header(tparquet::IndexPageHeader()); + auto bytes = serialize_page(index_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsUnknownAuxiliaryPage) { + tparquet::PageHeader unknown_header; + unknown_header.type = static_cast(127); + unknown_header.__set_compressed_page_size(3); + unknown_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(unknown_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); +} + +TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(1024); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + EXPECT_TRUE(should_cache_decompressed(&header, metadata, false)); + EXPECT_FALSE(should_cache_decompressed(&header, metadata, true)); + + EXPECT_TRUE(parquet_reader_compat("parquet-cpp version 1.5.0").data_page_v2_always_compressed); + EXPECT_FALSE(parquet_reader_compat("parquet-cpp version 2.0.0").data_page_v2_always_compressed); + EXPECT_TRUE(parquet_reader_compat("parquet-mr version 1.2.8").parquet_816_padding); + EXPECT_FALSE(parquet_reader_compat("parquet-mr version 1.2.9").parquet_816_padding); +} + +TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictMode) { + const int32_t overflow = 1000; + auto decode = [&](bool strict, MutableColumnPtr* column, IColumn::Filter* null_map) { + std::unique_ptr decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder)); + decoder->set_type_length(sizeof(overflow)); + Slice slice(reinterpret_cast(&overflow), sizeof(overflow)); + RETURN_IF_ERROR(decoder->set_data(&slice)); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + MutableColumnPtr column; + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ASSERT_TRUE(decode(false, &column, &null_map).ok()); + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + + null_map.clear(); + null_map.resize_fill(1, 0); + EXPECT_FALSE(decode(true, &column, &null_map).ok()); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryConversionFailureIsDeferredUntilReferenced) { + auto decode_dictionary_id = [](uint8_t dictionary_id, bool strict, IColumn::Filter* null_map, + MutableColumnPtr* column) { + const std::array dictionary_values {1, 1000}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + RETURN_IF_ERROR(Decoder::get_decoder(tparquet::Type::INT32, + tparquet::Encoding::RLE_DICTIONARY, decoder)); + decoder->set_type_length(sizeof(int32_t)); + RETURN_IF_ERROR( + decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size())); + char encoded_id[] = {1, 2, static_cast(dictionary_id)}; + Slice id_slice(encoded_id, sizeof(encoded_id)); + RETURN_IF_ERROR(decoder->set_data(&id_slice)); + + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + for (const bool strict : {false, true}) { + IColumn::Filter null_map; + IColumn::Filter* output_null_map = nullptr; + if (strict) { + null_map.resize_fill(1, 0); + output_null_map = &null_map; + } + + MutableColumnPtr column; + const auto unused_bad = decode_dictionary_id(0, strict, output_null_map, &column); + ASSERT_TRUE(unused_bad.ok()) << unused_bad; + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(assert_cast(*column).get_element(0), 1); + + const auto referenced_bad = decode_dictionary_id(1, strict, output_null_map, &column); + EXPECT_TRUE(referenced_bad.is()) << referenced_bad; + EXPECT_TRUE(column->empty()); + if (output_null_map != nullptr) { + EXPECT_EQ((*output_null_map)[0], 0); + } + } +} + +TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { + ColumnString column; + const std::string values = "aaabbbccc"; + column.insert_many_fixed_length_data(values.data(), 3, 3); + ASSERT_EQ(column.size(), 3); + EXPECT_EQ(column.get_data_at(0).to_string_view(), "aaa"); + EXPECT_EQ(column.get_data_at(1).to_string_view(), "bbb"); + EXPECT_EQ(column.get_data_at(2).to_string_view(), "ccc"); +} + +TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { + ColumnChunkReaderStatistics first_chunk; + first_chunk.page_read_counter = 1; + first_chunk.data_page_read_counter = 1; + ColumnChunkReaderStatistics second_chunk; + second_chunk.page_read_counter = 1; + second_chunk.data_page_read_counter = 1; + ColumnReader::ColumnStatistics combined; + ColumnReader::ColumnStatistics first(first_chunk, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0); + combined.merge(first); + combined.merge(second); + + EXPECT_EQ(combined.page_read_counter, 2); + ASSERT_EQ(combined.leaf_page_read_counters.size(), 2); + EXPECT_EQ(combined.leaf_page_read_counters[0], 1); + EXPECT_EQ(combined.leaf_page_read_counters[1], 1); +} + +TEST(ParquetV2NativeDecoderTest, NativePageCacheUsesStableFileDescriptionIdentity) { + ParquetPageCacheKeyBuilder first; + ParquetPageCacheKeyBuilder replaced; + first.init("s3://bucket/object::etag-v1::1234"); + replaced.init("s3://bucket/object::etag-v2::1234"); + EXPECT_EQ(first.make_key(4096, 128).fname, "s3://bucket/object::etag-v1::1234"); + EXPECT_NE(first.make_key(4096, 128).encode(), replaced.make_key(4096, 128).encode()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompressed) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(100); + header.__set_uncompressed_page_size(1000); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + + // The representation is explicit in V2; a codec and compression ratio cannot override it on + // a warm cache hit. + EXPECT_TRUE(should_cache_decompressed(&header, metadata)); +} + +TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { + EXPECT_FALSE(can_prepare_page_cache_payload(false, false, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, true, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, false, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, true, false)); + EXPECT_TRUE(can_prepare_page_cache_payload(true, false, true, true)); +} + +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { + ::doris::RowRanges row_ranges; + tparquet::ColumnChunk chunk; + ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); + + constexpr size_t max_retained_bytes = 64UL << 10; + reader.reserve_batch_scratch_for_test(1UL << 16); + const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); + ASSERT_GT(oversized_bytes, max_retained_bytes); + + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); + EXPECT_LT(released_bytes, oversized_bytes); + EXPECT_LE(released_bytes, max_retained_bytes + sizeof(void*)); +} + +} // namespace +} // namespace doris::format::parquet::native diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp new file mode 100644 index 00000000000000..490a0bde204a74 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "../../../benchmark/parquet/parquet_benchmark_scenarios.h" + +#include + +#include +#include +#include +#include + +namespace doris::parquet_benchmark { +namespace { + +TEST(ParquetBenchmarkScenariosTest, DecoderMatrixCoversNativeEncodingAndTypeFamilies) { + const auto scenarios = decoder_scenarios(); + const std::set> actual = [&] { + std::set> values; + for (const auto& scenario : scenarios) { + values.emplace(scenario.encoding, scenario.value_type); + } + return values; + }(); + + const std::set> expected { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; + EXPECT_EQ(actual, expected); +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversNullableSparseAndProjectionAxes) { + const auto scenarios = reader_scenarios(); + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == Encoding::PLAIN && + scenario.null_percent == null_percent && + scenario.null_pattern == pattern && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32; + })) << "missing nullable sparse axis combination"; + } + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversOperationsEncodingsAndSchemaWidths) { + const auto scenarios = reader_scenarios(); + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == operation; + })); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + for (const auto operation : {ReaderOperation::FULL_SCAN, ReaderOperation::PREDICATE_SCAN}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.encoding == encoding && scenario.operation == operation; + })); + } + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.schema_width == width && + scenario.predicate_position == predicate_position; + })); + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversFixedWidthRawFilterAxes) { + const auto scenarios = reader_scenarios(); + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == encoding && scenario.null_percent == 10 && + scenario.null_pattern == Pattern::ALTERNATING && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32 && + scenario.predicate_position == 0; + })) << "missing fixed-width raw filter axis combination"; + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, SelectionPlanDistinguishesClusteredAndSparseRuns) { + const auto clustered = make_selection_plan(1000, 10, Pattern::CLUSTERED); + EXPECT_EQ(clustered.total_rows, 1000); + EXPECT_EQ(clustered.selected_rows, 100); + ASSERT_EQ(clustered.ranges.size(), 1); + EXPECT_EQ(clustered.ranges.front().count, 100); + + const auto alternating = make_selection_plan(1000, 10, Pattern::ALTERNATING); + EXPECT_EQ(alternating.total_rows, 1000); + EXPECT_EQ(alternating.selected_rows, 100); + EXPECT_EQ(alternating.ranges.size(), 100); + + EXPECT_EQ(make_selection_plan(1000, 0, Pattern::ALTERNATING).ranges.size(), 0); + EXPECT_EQ(make_selection_plan(1000, 100, Pattern::ALTERNATING).ranges.size(), 1); +} + +} // namespace +} // namespace doris::parquet_benchmark diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp deleted file mode 100644 index fb4cd129e1b03d..00000000000000 --- a/be/test/format_v2/parquet/parquet_column_reader_test.cpp +++ /dev/null @@ -1,3682 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "core/types.h" -#include "format_v2/file_reader.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/selection_vector.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -template -const ColumnType& get_nullable_nested_column(const IColumn& column) { - // File-local schema exposed by the parquet reader follows Doris external-table semantics: - // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet - // field is required. - const auto& nullable_column = assert_cast(column); - return assert_cast(nullable_column.get_nested_column()); -} - -ParquetColumnSchema mock_column_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "mock"; - schema.type = std::make_shared(); - return schema; -} - -class BaseUnsupportedReader final : public ParquetColumnReader { -public: - BaseUnsupportedReader() - : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } -}; - -class DefaultSelectReader final : public ParquetColumnReader { -public: - DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - auto& values = assert_cast(*column); - for (int64_t row = 0; row < rows; ++row) { - values.insert_value(static_cast(_cursor + row)); - } - _cursor += rows; - *rows_read = rows; - _read_ranges.push_back(rows); - return Status::OK(); - } - - Status skip(int64_t rows) override { - _cursor += rows; - _skip_ranges.push_back(rows); - return Status::OK(); - } - - const std::vector& read_ranges() const { return _read_ranges; } - const std::vector& skip_ranges() const { return _skip_ranges; } - -private: - int64_t _cursor = 0; - std::vector _read_ranges; - std::vector _skip_ranges; -}; - -class NestedSkipReader final : public ParquetColumnReader { -public: - NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - *values_consumed = length_upper_bound; - return Status::OK(); - } -}; - -class ParquetColumnReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "reader.parquet").string(); - _plain_file_path = (_test_dir / "plain_reader.parquet").string(); - write_parquet_file(); - _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto metadata = _file_reader->metadata(); - ASSERT_EQ(metadata->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(_fields.size(), _expected_by_field.size()); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_all_null_int32_array() { - arrow::Int32Builder builder; - for (int64_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(builder.AppendNull().ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_required_struct_array() { - auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - const std::vector a_values = {101, 102, 103, 104, 105}; - const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; - for (size_t row = 0; row < a_values.size(); ++row) { - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); - EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(201).ok()); - EXPECT_TRUE(b_builder->Append("nsa").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(203).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(204).ok()); - EXPECT_TRUE(b_builder->Append("nsd").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_decimal_array() { - auto decimal_type = arrow::decimal128(38, 6); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto d_array_builder = std::make_unique( - decimal_type, arrow::default_memory_pool()); - field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* d_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(d_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - field_builders.push_back(list_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(305).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_map_array() { - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - field_builders.push_back(map_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(401).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("one").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(403).ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(404).ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(405).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->Append("five").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); - auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); - - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - std::vector> nested_field_builders; - nested_field_builders.push_back(list_builder); - auto nested_builder = std::make_shared( - nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); - std::vector> field_builders; - field_builders.push_back(nested_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(7).ok()); - EXPECT_TRUE(value_builder->Append(8).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), false))); - const std::vector> values = { - {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto value : row) { - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append(20).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(30).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(110).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(120).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(130).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_list_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - auto* a_builder = assert_cast(struct_builder->field_builder(0)); - auto* b_builder = assert_cast(struct_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(11).ok()); - EXPECT_TRUE(b_builder->Append("la").ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(12).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(13).ok()); - EXPECT_TRUE(b_builder->Append("ld").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(14).ok()); - EXPECT_TRUE(b_builder->Append("le").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_list_list_int_array() { - auto value_builder = std::make_shared(); - auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto inner_list_builder = std::make_shared( - arrow::default_memory_pool(), value_builder, inner_list_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, - arrow::list(arrow::field("element", inner_list_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->AppendNull().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - const std::vector>> values = { - {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, - {{7, "g"}}, {{8, "h"}, {9, "i"}}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto& [key, value] : row) { - EXPECT_TRUE(key_builder->Append(key).ok()); - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append("bb").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_struct_map_array() { - auto key_builder = std::make_shared(); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto value_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - auto* a_builder = assert_cast(value_builder->field_builder(0)); - auto* b_builder = assert_cast(value_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(21).ok()); - EXPECT_TRUE(b_builder->Append("ma").ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(22).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(24).ok()); - EXPECT_TRUE(b_builder->Append("me").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(201).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(key_builder->Append(202).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(203).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(204).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(205).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_map_list_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, - arrow::list(arrow::field("element", map_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("a").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(value_builder->Append("d").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_map_map_array() { - auto key_builder = std::make_shared(); - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(101).ok()); - EXPECT_TRUE(nested_value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(401).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_list_struct_map_list_array() { - auto element_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - element_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - auto map_builder = std::make_shared(arrow::default_memory_pool(), - key_builder, list_builder, map_type); - auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); - std::vector> struct_field_builders; - struct_field_builders.push_back(map_builder); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(10).ok()); - EXPECT_TRUE(element_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_map_list_map_array() { - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - nested_map_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(1).ok()); - EXPECT_TRUE(nested_value_builder->Append("a").ok()); - EXPECT_TRUE(nested_key_builder->Append(2).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(3).ok()); - EXPECT_TRUE(nested_value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(50).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(4).ok()); - EXPECT_TRUE(nested_value_builder->Append("d").ok()); - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, std::shared_ptr array, - std::function validator) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - _expected_by_field.push_back(std::move(validator)); - } - - void write_parquet_file() { - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); - add_field(arrow::field("all_null_int_col", arrow::int32(), true), - build_all_null_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - for (size_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(nullable_column.is_null_at(row)); - } - }); - add_field(arrow::field("struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false), - }), - false), - build_required_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); - const auto& struct_column = assert_cast(column); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - get_nullable_nested_column(struct_column.get_column(1)); - EXPECT_EQ(a_values.get_element(0), 101); - EXPECT_EQ(a_values.get_element(4), 105); - EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); - EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); - }); - add_field(arrow::field("nullable_struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }), - true), - build_nullable_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_nested = - assert_cast(b_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 201); - EXPECT_EQ(a_values.get_element(2), 203); - EXPECT_EQ(a_values.get_element(3), 204); - EXPECT_FALSE(b_values.is_null_at(0)); - EXPECT_TRUE(b_values.is_null_at(2)); - EXPECT_FALSE(b_values.is_null_at(3)); - EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); - }); - add_field(arrow::field("nullable_struct_decimal_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("d", arrow::decimal128(38, 6), true), - }), - true), - build_nullable_struct_with_decimal_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& d_values = - assert_cast(struct_column.get_column(1)); - const auto& d_nested = - assert_cast(d_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_FALSE(d_values.is_null_at(0)); - EXPECT_TRUE(d_values.is_null_at(2)); - EXPECT_FALSE(d_values.is_null_at(3)); - EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); - EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); - }); - auto struct_list_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), - true), - }); - add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), - build_nullable_struct_with_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_EQ(a_values.get_element(4), 305); - - const auto& xs_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = - assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = - assert_cast(xs_array.get_data()); - ASSERT_EQ(elements.size(), 4); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - const auto& values = - assert_cast(elements.get_nested_column()); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_EQ(values.get_element(3), 5); - }); - auto struct_map_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("kv", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - }); - add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), - build_nullable_struct_with_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(2), 403); - EXPECT_EQ(a_values.get_element(3), 404); - EXPECT_EQ(a_values.get_element(4), 405); - - const auto& kv_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = - assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); - }); - auto nested_struct_list_type = arrow::struct_({ - arrow::field("nested", - arrow::struct_({ - arrow::field("xs", - arrow::list(arrow::field("element", - arrow::int32(), true)), - true), - }), - true), - }); - add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, - true), - build_nullable_struct_with_nested_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = - assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); - }); - add_field(arrow::field("list_int_col", - arrow::list(arrow::field("element", arrow::int32(), false)), false), - build_required_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); - const auto* array_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ( - remove_nullable(array_type->get_nested_type())->get_primitive_type(), - TYPE_INT); - const auto& array_column = assert_cast(column); - ASSERT_EQ(array_column.size(), ROW_COUNT); - const auto array_size_at = [&array_column](size_t row_idx) { - return array_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(array_size_at(0), 2); - EXPECT_EQ(array_size_at(1), 1); - EXPECT_EQ(array_size_at(2), 3); - EXPECT_EQ(array_size_at(4), 2); - const auto& values = - get_nullable_nested_column(array_column.get_data()); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(5), 6); - EXPECT_EQ(values.get_element(8), 9); - }); - add_field(arrow::field("nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), true), - build_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 30); - EXPECT_EQ(values.get_element(4), 40); - }); - add_field(arrow::field("required_nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), false), - build_required_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& array_column = assert_cast(column); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 5); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - ASSERT_EQ(elements.size(), 5); - EXPECT_TRUE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(4)); - }); - auto list_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field("nullable_list_struct_col", - arrow::list(arrow::field("element", list_struct_type, true)), true), - build_nullable_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - - const auto& elements = - assert_cast(array_column.get_data()); - const auto& struct_column = - assert_cast(elements.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 11); - EXPECT_EQ(a_values.get_element(1), 12); - EXPECT_EQ(a_values.get_element(3), 13); - EXPECT_EQ(a_values.get_element(4), 14); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); - }); - auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field(arrow::field("nullable_list_list_int_col", - arrow::list(arrow::field("element", nested_list_type, true)), true), - build_nullable_list_list_int_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 4); - EXPECT_EQ(outer_offsets[3], 5); - EXPECT_EQ(outer_offsets[4], 7); - - const auto& inner_nullable = - assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_FALSE(inner_nullable.is_null_at(6)); - - const auto& inner_array = - assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[5], 5); - EXPECT_EQ(inner_offsets[6], 7); - - const auto& elements = - assert_cast(inner_array.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 7); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 3); - EXPECT_EQ(values.get_element(4), 4); - EXPECT_EQ(values.get_element(5), 5); - EXPECT_TRUE(elements.is_null_at(6)); - }); - add_field(arrow::field( - "map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), - false), - build_required_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); - const auto* map_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), - TYPE_INT); - EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), - TYPE_STRING); - const auto& map_column = assert_cast(column); - ASSERT_EQ(map_column.size(), ROW_COUNT); - const auto map_size_at = [&map_column](size_t row_idx) { - return map_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(map_size_at(0), 2); - EXPECT_EQ(map_size_at(1), 1); - EXPECT_EQ(map_size_at(2), 3); - EXPECT_EQ(map_size_at(4), 2); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(keys.size(), 9); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(5), 6); - EXPECT_EQ(keys.get_element(8), 9); - EXPECT_EQ(values.get_data_at(0).to_string(), "a"); - EXPECT_EQ(values.get_data_at(5).to_string(), "f"); - EXPECT_EQ(values.get_data_at(8).to_string(), "i"); - }); - add_field( - arrow::field("nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - build_nullable_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 10); - EXPECT_EQ(keys.get_element(1), 20); - EXPECT_EQ(keys.get_element(3), 40); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(values.is_null_at(3)); - }); - add_field( - arrow::field("required_nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - false), - build_required_nullable_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& map_column = assert_cast(column); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 4); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(3)); - }); - auto map_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field( - "nullable_map_int_struct_col", - arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), - true), - build_nullable_int_struct_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& struct_column = - assert_cast(values.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 21); - EXPECT_EQ(a_values.get_element(1), 22); - EXPECT_EQ(a_values.get_element(3), 24); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); - }); - auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field( - arrow::field("nullable_map_int_list_col", - arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), - true), - build_nullable_int_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), ROW_COUNT); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 4); - EXPECT_EQ(map_offsets[4], 5); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 5); - EXPECT_EQ(keys.get_element(0), 201); - EXPECT_EQ(keys.get_element(1), 202); - EXPECT_EQ(keys.get_element(2), 203); - EXPECT_EQ(keys.get_element(3), 204); - EXPECT_EQ(keys.get_element(4), 205); - - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_FALSE(values.is_null_at(4)); - - const auto& list_column = - assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); - - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(element_values.get_element(0), 1); - EXPECT_EQ(element_values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(element_values.get_element(3), 3); - EXPECT_EQ(element_values.get_element(4), 4); - }); - auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field("nullable_list_map_int_string_col", - arrow::list(arrow::field("element", list_map_type, true)), true), - build_nullable_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& map_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 3); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); - }); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field( - "nullable_map_int_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), - true), - build_nullable_int_map_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 4); - - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 4); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - - const auto& inner_values = - assert_cast(outer_map.get_values()); - ASSERT_EQ(inner_values.size(), 4); - EXPECT_FALSE(inner_values.is_null_at(0)); - EXPECT_FALSE(inner_values.is_null_at(1)); - EXPECT_TRUE(inner_values.is_null_at(2)); - EXPECT_FALSE(inner_values.is_null_at(3)); - - const auto& inner_map = - assert_cast(inner_values.get_nested_column()); - const auto& inner_offsets = inner_map.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 4); - EXPECT_EQ(inner_offsets[0], 1); - EXPECT_EQ(inner_offsets[1], 1); - EXPECT_EQ(inner_offsets[2], 1); - EXPECT_EQ(inner_offsets[3], 2); - const auto& inner_keys = - get_nullable_nested_column(inner_map.get_keys()); - const auto& inner_strings = - assert_cast(inner_map.get_values()); - const auto& inner_string_data = - assert_cast(inner_strings.get_nested_column()); - ASSERT_EQ(inner_keys.size(), 2); - EXPECT_EQ(inner_keys.get_element(0), 101); - EXPECT_EQ(inner_keys.get_element(1), 401); - EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(inner_strings.is_null_at(1)); - }); - auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto deep_list_map_type = - arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); - auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); - add_field(arrow::field("nullable_list_struct_map_list_col", - arrow::list(arrow::field("element", deep_list_struct_type, true)), - true), - build_deep_list_struct_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& struct_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - - const auto& struct_column = - assert_cast(struct_values.get_nested_column()); - const auto& map_values = - assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - - const auto& lists = - assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 4); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 4); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 3); - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 3); - EXPECT_EQ(element_values.get_element(0), 10); - EXPECT_TRUE(elements.is_null_at(1)); - EXPECT_EQ(element_values.get_element(2), 40); - }); - auto deep_map_nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto deep_map_list_type = - arrow::list(arrow::field("element", deep_map_nested_map_type, true)); - add_field( - arrow::field( - "nullable_map_int_list_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), - true), - build_deep_map_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); - - const auto& inner_maps = - assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 6); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_FALSE(inner_maps.is_null_at(1)); - EXPECT_TRUE(inner_maps.is_null_at(2)); - EXPECT_FALSE(inner_maps.is_null_at(3)); - EXPECT_TRUE(inner_maps.is_null_at(4)); - EXPECT_FALSE(inner_maps.is_null_at(5)); - const auto& inner_map_column = - assert_cast(inner_maps.get_nested_column()); - const auto& inner_offsets = inner_map_column.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 6); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 3); - EXPECT_EQ(inner_offsets[4], 3); - EXPECT_EQ(inner_offsets[5], 4); - const auto& inner_keys = - get_nullable_nested_column(inner_map_column.get_keys()); - ASSERT_EQ(inner_keys.size(), 4); - EXPECT_EQ(inner_keys.get_element(0), 1); - EXPECT_EQ(inner_keys.get_element(1), 2); - EXPECT_EQ(inner_keys.get_element(2), 3); - EXPECT_EQ(inner_keys.get_element(3), 4); - const auto& strings = - assert_cast(inner_map_column.get_values()); - const auto& string_data = - assert_cast(strings.get_nested_column()); - ASSERT_EQ(strings.size(), 4); - EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(strings.is_null_at(1)); - EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); - }); - - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - - auto file_result = arrow::io::FileOutputStream::Open(_file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ROW_COUNT, builder.build())); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_plain_reader(size_t field_idx) { - // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy - // because Arrow BinaryRecordReader has a stricter reset contract than - // DictionaryRecordReader. - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); - DORIS_CHECK(plain_file_result.ok()); - std::shared_ptr plain_out = *plain_file_result; - ::parquet::WriterProperties::Builder plain_builder; - plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - plain_builder.compression(::parquet::Compression::UNCOMPRESSED); - plain_builder.disable_dictionary(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); - DORIS_CHECK(plain_out->Close().ok()); - - _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); - auto metadata = _plain_file_reader->metadata(); - DORIS_CHECK(metadata != nullptr); - DORIS_CHECK(metadata->num_row_groups() == 1); - _plain_row_group = _plain_file_reader->RowGroup(0); - DORIS_CHECK(_plain_row_group != nullptr); - - ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_child_reader(size_t field_idx, - size_t child_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[child_idx]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_grandchild_reader( - size_t field_idx, size_t child_idx, size_t grandchild_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - const auto& child_schema = *struct_schema.children[child_idx]; - EXPECT_LT(grandchild_idx, child_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = child_schema.local_id; - child_projection.project_all_children = false; - format::LocalColumnIndex grandchild_projection; - grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; - child_projection.children.push_back(std::move(grandchild_projection)); - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - void read_and_validate(size_t field_idx) const { - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - _expected_by_field[field_idx](*_fields[field_idx], *column); - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet test field " << name; - return _fields.size(); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::string _plain_file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; - std::vector> _expected_by_field; -}; - -TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { - SelectionVector identity; - ASSERT_TRUE(identity.verify(4, 5).ok()); - auto ranges = selection_to_ranges(identity, 4); - ASSERT_EQ(ranges.size(), 1); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 4); - - std::array selected = {0, 2, 3, 6, 6}; - SelectionVector external(selected.data(), 4); - auto status = external.verify(3, 7); - ASSERT_TRUE(status.ok()) << status; - ranges = selection_to_ranges(external, 3); - ASSERT_EQ(ranges.size(), 2); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 1); - EXPECT_EQ(ranges[1].start, 2); - EXPECT_EQ(ranges[1].length, 2); - - EXPECT_FALSE(external.verify(8, 7).ok()); - EXPECT_FALSE(external.verify(5, 7).ok()); - EXPECT_FALSE(external.verify(4, 6).ok()); - - std::array duplicate = {0, 2, 2}; - SelectionVector non_strict(duplicate.data(), duplicate.size()); - EXPECT_FALSE(non_strict.verify(3, 5).ok()); - EXPECT_FALSE(identity.verify(1, -1).ok()); -} - -TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { - DefaultSelectReader reader; - std::array selected = {1, 3, 4}; - SelectionVector selection(selected.data(), selected.size()); - auto column = ColumnInt32::create(); - MutableColumnPtr mutable_column = std::move(column); - auto status = reader.select(selection, selected.size(), 6, mutable_column); - ASSERT_TRUE(status.ok()) << status; - - const auto& values = assert_cast(*mutable_column); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 3); - EXPECT_EQ(values.get_element(2), 4); - EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); - EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); - - BaseUnsupportedReader unsupported_reader; - auto skip_status = unsupported_reader.skip(1); - EXPECT_FALSE(skip_status.ok()); - EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); - EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); - int64_t values_read = 0; - EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); - EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); - - NestedSkipReader nested_reader; - auto nested_status = nested_reader.consume_nested_column(3, &values_read); - ASSERT_TRUE(nested_status.ok()) << nested_status; - EXPECT_EQ(values_read, 3); -} - -TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { - read_and_validate(find_field_idx("int32_col")); - read_and_validate(find_field_idx("string_col")); - read_and_validate(find_field_idx("nullable_int_col")); - read_and_validate(find_field_idx("all_null_int_col")); - - auto reader = create_reader(find_field_idx("int32_col")); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 3); - const auto& values = assert_cast(*column); - ASSERT_EQ(values.size(), ROW_COUNT); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_EQ(values.get_element(2), 30); - EXPECT_EQ(values.get_element(4), 50); -} - -TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(0).ok()); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& first_value = assert_cast(*column); - EXPECT_EQ(first_value.get_element(0), 10); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(2).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& skipped_values = assert_cast(*column); - EXPECT_EQ(skipped_values.get_element(0), 30); - EXPECT_EQ(skipped_values.get_element(1), 40); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - EXPECT_EQ(rows_read, 0); - EXPECT_EQ(column->size(), 0); - - reader = create_reader(find_field_idx("nullable_int_col")); - ASSERT_TRUE(reader->skip(1).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); -} - -TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector all_selected(ROW_COUNT); - auto column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); - const auto& all_values = assert_cast(*column); - ASSERT_EQ(all_values.size(), ROW_COUNT); - EXPECT_EQ(all_values.get_element(0), 10); - EXPECT_EQ(all_values.get_element(4), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array disjoint = {0, 2, 4}; - SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); - const auto& disjoint_values = assert_cast(*column); - ASSERT_EQ(disjoint_values.size(), 3); - EXPECT_EQ(disjoint_values.get_element(0), 10); - EXPECT_EQ(disjoint_values.get_element(1), 30); - EXPECT_EQ(disjoint_values.get_element(2), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array single = {2}; - SelectionVector single_selection(single.data(), single.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); - const auto& single_value = assert_cast(*column); - ASSERT_EQ(single_value.size(), 1); - EXPECT_EQ(single_value.get_element(0), 30); - - reader = create_reader(find_field_idx("int32_col")); - std::array first_last = {0, 4}; - SelectionVector first_last_selection(first_last.data(), first_last.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); - const auto& first_last_values = assert_cast(*column); - ASSERT_EQ(first_last_values.size(), 2); - EXPECT_EQ(first_last_values.get_element(0), 10); - EXPECT_EQ(first_last_values.get_element(1), 50); - - reader = create_reader(find_field_idx("int32_col")); - SelectionVector empty_selection; - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); - ASSERT_EQ(column->size(), 0); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& after_empty_select = assert_cast(*column); - ASSERT_EQ(after_empty_select.size(), 1); - EXPECT_EQ(after_empty_select.get_element(0), 30); - - reader = create_reader(find_field_idx("nullable_int_col")); - std::array nullable_rows = {0, 1, 2}; - SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& int_schema = *_fields[find_field_idx("int32_col")]; - ParquetColumnSchema invalid_leaf; - invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; - invalid_leaf.name = "invalid_leaf"; - invalid_leaf.type = int_schema.type; - invalid_leaf.type_descriptor = int_schema.type_descriptor; - invalid_leaf.descriptor = int_schema.descriptor; - invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); - auto status = factory.create(invalid_leaf, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); - - ParquetColumnSchema null_descriptor; - null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; - null_descriptor.name = "null_descriptor"; - null_descriptor.type = int_schema.type; - null_descriptor.type_descriptor = int_schema.type_descriptor; - null_descriptor.leaf_column_id = int_schema.leaf_column_id; - status = factory.create(null_descriptor, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); - - const auto& list_element_schema = - *_fields[find_field_idx("nullable_list_int_col")]->children[0]; - status = factory.create(list_element_schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex projection = - format::LocalColumnIndex::partial_local(list_schema.local_id); - format::LocalColumnIndex element_projection = - format::LocalColumnIndex::partial_local(list_element_schema.local_id); - projection.children.push_back(std::move(element_projection)); - status = factory.create(list_schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& struct_schema = *_fields[find_field_idx("struct_col")]; - format::LocalColumnIndex struct_empty = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - auto status = factory.create(struct_schema, &struct_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); - - format::LocalColumnIndex struct_invalid = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(struct_schema, &struct_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex list_empty = - format::LocalColumnIndex::partial_local(list_schema.local_id); - status = factory.create(list_schema, &list_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); - - const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; - const auto& value_schema = *map_schema.children[1]; - format::LocalColumnIndex map_invalid = - format::LocalColumnIndex::partial_local(map_schema.local_id); - map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(map_schema, &map_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { - read_and_validate(find_field_idx("struct_col")); - read_and_validate(find_field_idx("nullable_struct_col")); - read_and_validate(find_field_idx("nullable_struct_decimal_col")); - read_and_validate(find_field_idx("list_int_col")); - read_and_validate(find_field_idx("nullable_list_int_col")); - read_and_validate(find_field_idx("required_nullable_list_int_col")); - read_and_validate(find_field_idx("nullable_list_struct_col")); - read_and_validate(find_field_idx("nullable_list_list_int_col")); - read_and_validate(find_field_idx("map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_string_col")); - read_and_validate(find_field_idx("required_nullable_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_struct_col")); - read_and_validate(find_field_idx("nullable_map_int_list_col")); - read_and_validate(find_field_idx("nullable_list_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); - read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); - read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); -} - -TEST_F(ParquetColumnReaderTest, SkipThenRead) { - auto reader = create_reader(find_field_idx("int32_col")); - auto st = reader->skip(2); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 2); - EXPECT_EQ(int_values.get_element(0), 30); - EXPECT_EQ(int_values.get_element(1), 40); -} - -TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 2); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 3); - EXPECT_EQ(int_values.get_element(0), 10); - EXPECT_EQ(int_values.get_element(1), 30); - EXPECT_EQ(int_values.get_element(2), 50); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { - const auto field_idx = find_field_idx("struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& struct_column = assert_cast(*column); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); - EXPECT_EQ(values.get_data_at(4).to_string(), "se"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { - const auto field_idx = find_field_idx("nullable_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& nullable_column = assert_cast(*column); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = assert_cast(struct_column.get_column(0)); - const auto& nested_values = assert_cast(values.get_nested_column()); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& list_schema = *_fields[field_idx]; - ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); - ASSERT_EQ(list_schema.children.size(), 1); - const auto& element_schema = *list_schema.children[0]; - ASSERT_EQ(element_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = list_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex element_projection; - element_projection.index = element_schema.local_id; - element_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = element_schema.children[1]->local_id; - element_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(element_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(list_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* array_type = - assert_cast(remove_nullable(reader->type()).get()); - const auto* element_type = assert_cast( - remove_nullable(array_type->get_nested_type()).get()); - ASSERT_EQ(element_type->get_elements().size(), 1); - EXPECT_EQ(element_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& elements = assert_cast(array_column.get_data()); - const auto& struct_column = assert_cast(elements.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); - ASSERT_EQ(map_schema.children.size(), 2); - const auto& value_schema = *map_schema.children[1]; - ASSERT_EQ(value_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = map_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex value_projection; - value_projection.index = value_schema.local_id; - value_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = value_schema.children[1]->local_id; - value_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(value_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); - const auto* value_type = - assert_cast(remove_nullable(map_type->get_value_type()).get()); - ASSERT_EQ(value_type->get_elements().size(), 1); - EXPECT_EQ(value_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - const auto& values = assert_cast(map_column.get_values()); - const auto& struct_column = assert_cast(values.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); -} - -TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - const auto& value_schema = *map_schema.children[1]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_NE(reader, nullptr); -} - -TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "xs"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = assert_cast(xs_array.get_data()); - const auto& values = assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 4); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 5); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "kv"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); - auto reader = create_projected_grandchild_reader(field_idx, 0, 0); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(1), 304); - EXPECT_EQ(a_values.get_element(2), 305); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(1), 404); - EXPECT_EQ(a_values.get_element(2), 405); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 1); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 1); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 1); - EXPECT_EQ(inner_offsets[0], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 5); - EXPECT_EQ(outer_offsets[2], 7); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[6], 7); -} - -TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_plain_reader(field_idx); - - // Row 0 contains two STRING values. The levels-only skip must release (and discard) those - // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the - // next batch, ParquetLeafReader observes more values than current definition/repetition levels. - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 3); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(map_column.get_offsets()[1], 0); - EXPECT_EQ(map_column.get_offsets()[2], 1); - const auto& values = get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(values.size(), 1); - EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); -} - -TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 2); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 4); - EXPECT_EQ(map_offsets[2], 5); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(4)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 3); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_FALSE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 3); - EXPECT_TRUE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_FALSE(map_values.is_null_at(2)); - - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 2); - EXPECT_EQ(keys.get_element(0), 3); - EXPECT_EQ(keys.get_element(1), 4); - const auto& lists = assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 2); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 3); - EXPECT_EQ(outer_keys.get_element(0), 30); - EXPECT_EQ(outer_keys.get_element(1), 40); - EXPECT_EQ(outer_keys.get_element(2), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 3); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_FALSE(lists.is_null_at(2)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 3); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); - EXPECT_EQ(list_offsets[2], 3); - const auto& inner_maps = assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 3); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_TRUE(inner_maps.is_null_at(1)); - EXPECT_FALSE(inner_maps.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp deleted file mode 100644 index 0d0f9a2f8567cc..00000000000000 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ /dev/null @@ -1,506 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" - -namespace doris::format::parquet { -namespace { - -std::shared_ptr fixed_binary_array(const std::vector& values, - int byte_width) { - auto type = arrow::fixed_size_binary(byte_width); - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); -} - -struct CapturedDecodedView { - DecodedValueKind value_kind = DecodedValueKind::INT32; - DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; - int64_t row_count = 0; - int decimal_precision = -1; - int decimal_scale = -1; - int fixed_length = -1; - bool timestamp_is_adjusted_to_utc = false; - bool enable_strict_mode = false; - const cctz::time_zone* timezone = nullptr; - bool null_map_is_null = true; - std::vector null_map; - std::vector fixed_values; - std::vector binary_values; - std::vector owned_binary_values; -}; - -ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, - CapturedDecodedView* captured, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false) { - auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { - captured->value_kind = view.value_kind; - captured->time_unit = view.time_unit; - captured->row_count = view.row_count; - captured->decimal_precision = view.decimal_precision; - captured->decimal_scale = view.decimal_scale; - captured->fixed_length = view.fixed_length; - captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; - captured->enable_strict_mode = view.enable_strict_mode; - captured->timezone = view.timezone; - captured->null_map_is_null = view.null_map == nullptr; - captured->null_map.clear(); - if (view.null_map != nullptr) { - captured->null_map.assign(view.null_map, view.null_map + view.row_count); - } - captured->fixed_values.clear(); - if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 8); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } - captured->binary_values.clear(); - captured->owned_binary_values.clear(); - if (view.binary_values != nullptr) { - captured->owned_binary_values.reserve(view.binary_values->size()); - for (const auto& value : *view.binary_values) { - captured->owned_binary_values.emplace_back( - value.data == nullptr ? std::string() - : std::string(value.data, value.size)); - } - captured->binary_values.reserve(captured->owned_binary_values.size()); - for (const auto& value : captured->owned_binary_values) { - captured->binary_values.emplace_back(value.data(), value.size()); - } - } - return Status::OK(); - }; - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, - enable_strict_mode, std::move(appender)); -} - -} // namespace - -struct ParquetLeafReaderTestAccess { - static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, - const std::vector& rep_levels, - const std::vector& values, - bool read_dense_for_nullable = false) { - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._consumed_level_count = static_cast(def_levels.size()); - batch._decoded_level_count = static_cast(def_levels.size()); - batch._values_written = static_cast(values.size()); - batch._def_levels = def_levels.data(); - batch._rep_levels = rep_levels.data(); - batch._fixed_values = reinterpret_cast(values.data()); - batch._read_dense_for_nullable = read_dense_for_nullable; - return batch; - } - - static Status build_nested_batch(const ParquetLeafReader& reader, - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, - int16_t value_slot_repetition_level, - ParquetNestedScalarBatch* nested_batch) { - return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, - value_slot_definition_level, nested_batch, - value_slot_repetition_level); - } -}; - -std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, - int16_t max_repetition_level) { - auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32); - return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, - max_repetition_level); -} - -ParquetLeafReader make_nested_leaf_reader( - const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { - ParquetTypeDescriptor type_descriptor; - type_descriptor.physical_type = ::parquet::Type::INT32; - type_descriptor.doris_type = type; - return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", - nullptr); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30, 50}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - ASSERT_EQ(nullable.size(), 5); - EXPECT_FALSE(nullable.is_null_at(0)); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_FALSE(nullable.is_null_at(2)); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_FALSE(nullable.is_null_at(4)); - const auto& nested = assert_cast(nullable.get_nested_column()); - EXPECT_EQ(nested.get_element(0), 10); - EXPECT_EQ(nested.get_element(2), 30); - EXPECT_EQ(nested.get_element(4), 50); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - descriptor.fixed_length = 2; - auto type = std::make_shared(); - auto reader = make_leaf_reader(descriptor, type); - - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::FIXED_BINARY; - batch._binary_chunks = {fixed_binary_array( - {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; - batch._values_written = 5; - - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, nullptr, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& floats = assert_cast(*column); - ASSERT_EQ(floats.size(), 5); - EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); - EXPECT_TRUE(std::signbit(floats.get_element(1))); - EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); - EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); - EXPECT_TRUE(std::isnan(floats.get_element(4))); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("aa").ok()); - ASSERT_TRUE(builder.Append("cc").ok()); - ASSERT_TRUE(builder.Append("ee").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 3; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - const auto& strings = assert_cast(nullable.get_nested_column()); - ASSERT_EQ(nullable.size(), 5); - EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("only_one").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 1; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 3, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), - std::string::npos); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT64; - descriptor.time_unit = ParquetTimeUnit::NANOS; - descriptor.decimal_precision = 18; - descriptor.decimal_scale = 4; - descriptor.fixed_length = 12; - descriptor.timestamp_is_adjusted_to_utc = true; - auto type = make_nullable(std::make_shared()); - cctz::time_zone shanghai; - ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - - CapturedDecodedView captured; - auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); - const std::vector values = {100, 200, 300}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT64; - batch._fixed_values = reinterpret_cast(values.data()); - batch._values_written = values.size(); - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); - EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); - EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); - EXPECT_EQ(captured.row_count, 3); - EXPECT_EQ(captured.decimal_precision, 18); - EXPECT_EQ(captured.decimal_scale, 4); - EXPECT_EQ(captured.fixed_length, 12); - EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); - EXPECT_TRUE(captured.enable_strict_mode); - EXPECT_EQ(captured.timezone, &shanghai); - EXPECT_FALSE(captured.null_map_is_null); - EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); - - auto required_column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); - - const NullMap empty_null_map; - ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { - ParquetTypeDescriptor binary_descriptor; - binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - binary_descriptor.fixed_length = 4; - auto type = std::make_shared(); - - CapturedDecodedView binary_view; - auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); - ParquetLeafBatch binary_batch; - binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; - binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; - binary_batch._values_written = 2; - auto binary_column = type->create_column(); - ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); - EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); - EXPECT_EQ(binary_view.fixed_length, 4); - ASSERT_EQ(binary_view.owned_binary_values.size(), 2); - EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); - EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); - - ParquetTypeDescriptor float16_descriptor; - float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - float16_descriptor.fixed_length = 2; - CapturedDecodedView float16_view; - auto float16_reader = make_spy_leaf_reader(float16_descriptor, - std::make_shared(), &float16_view); - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - ParquetLeafBatch float16_batch; - float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; - float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; - float16_batch._values_written = 2; - auto float16_column = std::make_shared()->create_column(); - ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); - EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); - ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); - const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); - EXPECT_FLOAT_EQ(floats[0], 1.5F); - EXPECT_FLOAT_EQ(floats[1], 2.0F); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 2, 2}; - const std::vector rep_levels = {0, 1, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.records_read, 2); - EXPECT_EQ(nested_batch.levels_written, 3); - EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); - const auto& nested_values = assert_cast(*nested_batch.values_column); - ASSERT_EQ(nested_values.size(), 3); - EXPECT_EQ(nested_values.get_element(0), 10); - EXPECT_EQ(nested_values.get_element(2), 30); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 777, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {1, 2, 0, 2}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {777, 10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 0, 2, 0}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { - auto descriptor = int32_column_descriptor(1, 0); - auto reader = - make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); - const std::vector def_levels = {1}; - const std::vector rep_levels = {0}; - const std::vector values = {10}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), - std::string::npos); -} - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index 940f94a373a013..cb54cab86b27d8 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -26,134 +26,6 @@ namespace doris::format::parquet { namespace { -void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, - const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, - int64_t output_offset, int64_t copy_size) { - EXPECT_EQ(entry.cached_range.offset, cached_range.offset); - EXPECT_EQ(entry.cached_range.size, cached_range.size); - EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); - EXPECT_EQ(entry.output_offset, output_offset); - EXPECT_EQ(entry.copy_size, copy_size); -} - -TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { - const std::vector cached_ranges = { - {100, 100}, - }; - - // Request [120, 150) is fully inside cached [100, 200). The reader should lookup - // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. - auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); - - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { - const std::vector cached_ranges = { - {180, 80}, - {100, 80}, - }; - - // Request [100, 260) is larger than either cached entry, but the two cached ranges - // exactly cover it. The copy plan stitches the two exact cache entries together. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); - expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { - const std::vector cached_ranges = { - {150, 110}, - {100, 100}, - }; - - // Request [100, 260) is covered by overlapping cached ranges. The first copy uses - // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); - expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); -} - -TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { - const std::vector cached_ranges = { - {100, 80}, - {200, 60}, - }; - - // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. - // The caller must read the whole request from the file instead of returning - // a partially cached result. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { - const std::vector cached_ranges = { - {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, - }; - - // Only [120, 150) intersects the request, but it does not cover the request start - // [100, 120), so this is still a miss. - auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { - const std::vector cached_ranges = { - {100, 100}, - }; - - EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); -} - -TEST(ParquetPageCacheRangeTest, PerFileRangeIndexDeduplicatesAndEvictsAtCapacity) { - detail::ParquetPageCacheRangeIndex index(3); - index.insert({200, 20}); - index.insert({100, 30}); - index.insert({100, 10}); - index.insert({100, 30}); - - EXPECT_EQ(index.size(), 3); - index.insert({300, 40}); - const auto ranges = index.ranges(); - ASSERT_EQ(ranges.size(), 3); - EXPECT_EQ(ranges[0].offset, 100); - EXPECT_EQ(ranges[0].size, 30); - EXPECT_EQ(ranges[1].offset, 200); - EXPECT_EQ(ranges[1].size, 20); - EXPECT_EQ(ranges[2].offset, 300); - - index.erase({100, 30}); - EXPECT_EQ(index.size(), 2); -} - -TEST(ParquetPageCacheRangeTest, DirectorySharesBoundedIndexAcrossReaderLifetimes) { - detail::ParquetPageCacheRangeDirectory directory(2); - auto first_reader_index = directory.get_or_create("file-a"); - first_reader_index->insert({100, 100}); - first_reader_index.reset(); - - // The directory owns the per-file index, so reader B still discovers reader A's wider cache - // entry and can plan a subset hit after A closes. - auto second_reader_index = directory.get_or_create("file-a"); - const auto plan = detail::plan_page_cache_range_read(120, 30, second_reader_index->ranges()); - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); - - directory.get_or_create("file-b"); - directory.get_or_create("file-c"); - EXPECT_EQ(directory.size(), 2); -} - TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, @@ -173,6 +45,19 @@ TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) EXPECT_EQ(valid_ranges[1].size, 60); } +TEST(ParquetPageCacheRangeTest, SerializedIndexesAreBoundedIndividuallyAndWhenCoalesced) { + constexpr size_t file_size = 1ULL << 30; + const auto budget = detail::MAX_SERIALIZED_PARQUET_INDEX_BYTES; + + EXPECT_TRUE(detail::is_serialized_index_range_safe(file_size, 0, budget)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, 0, budget + 1)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, -1, 1)); + + EXPECT_TRUE(detail::is_serialized_index_span_safe(100, 100 + budget)); + // Individually small adjacent indexes must not combine into one unbounded allocation. + EXPECT_FALSE(detail::is_serialized_index_span_safe(100, 101 + budget)); +} + TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { const std::vector ranges = { {0, 512}, diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 36b7cebdaa9cb9..75e6545906f263 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -5,9 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,188 +13,49 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include #include #include #include #include -#include #include #include "common/consts.h" #include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" +#include "core/block/block.h" #include "core/column/column_string.h" -#include "core/column/column_struct.h" #include "core/column/column_vector.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "format_v2/parquet/selection_vector.h" #include "storage/utils.h" namespace doris::format::parquet { namespace { -ParquetColumnSchema int64_schema(std::string name = "mock") { +ParquetColumnSchema int64_schema() { ParquetColumnSchema schema; schema.local_id = 0; - schema.name = std::move(name); + schema.name = "mock"; schema.type = std::make_shared(); return schema; } -ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, - int16_t definition_level, int16_t repetition_level = 0, - int16_t repeated_ancestor_definition_level = 0) { - ParquetColumnSchema schema = int64_schema(std::move(name)); - schema.type = make_nullable(std::make_shared()); - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - return schema; -} - -ParquetColumnSchema nested_struct_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "struct"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})); - return schema; -} - -ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, - int16_t nullable_definition_level, int16_t definition_level, - int16_t repetition_level, - int16_t repeated_ancestor_definition_level) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.kind = ParquetColumnSchemaKind::LIST; - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - schema.type = make_nullable(std::make_shared(std::move(element_type))); - return schema; -} - -ParquetColumnSchema nested_map_schema( - DataTypePtr value_type = make_nullable(std::make_shared())) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "map"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.repetition_level = 1; - schema.repeated_ancestor_definition_level = 2; - schema.type = make_nullable(std::make_shared( - make_nullable(std::make_shared()), std::move(value_type))); - return schema; -} - -ParquetColumnSchema bare_repeated_int64_list_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "repeated"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.definition_level = 1; - schema.repetition_level = 1; - schema.repeated_repetition_level = 1; - schema.repeated_ancestor_definition_level = 1; - schema.type = std::make_shared(std::make_shared()); - return schema; -} - -std::unique_ptr primitive_child(int local_id, std::string name, - DataTypePtr type) { - auto child = std::make_unique(); - child->local_id = local_id; - child->name = std::move(name); - child->kind = ParquetColumnSchemaKind::PRIMITIVE; - child->leaf_column_id = local_id; - child->type = std::move(type); - child->type_descriptor.physical_type = ::parquet::Type::INT32; - child->type_descriptor.doris_type = child->type; - return child; -} - -ParquetColumnSchema struct_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "s"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.children.push_back(primitive_child(0, "a", std::make_shared())); - schema.children.push_back(primitive_child(1, "b", std::make_shared())); - DataTypes types = {make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)}; - Strings names = {"a", "b"}; - schema.type = std::make_shared(types, names); - return schema; -} - -ParquetColumnSchema list_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "xs"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.children.push_back(primitive_child(0, "element", std::make_shared())); - schema.type = std::make_shared(schema.children[0]->type); - return schema; -} - -ParquetColumnSchema map_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "m"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.children.push_back(primitive_child(0, "key", std::make_shared())); - schema.children.push_back(primitive_child(1, "value", std::make_shared())); - schema.type = std::make_shared(make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)); - return schema; -} - class CursorColumnReader final : public ParquetColumnReader { public: CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("invalid mock read arguments"); - } - auto* values = assert_cast(column.get()); + DORIS_CHECK(column); + DORIS_CHECK(rows_read != nullptr); + auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { - values->insert_value(_cursor + row); + values.insert_value(_cursor + row); } _read_lengths.push_back(rows); _cursor += rows; @@ -205,271 +64,33 @@ class CursorColumnReader final : public ParquetColumnReader { } Status skip(int64_t rows) override { + DORIS_CHECK(rows >= 0); _skip_lengths.push_back(rows); _cursor += rows; return Status::OK(); } + void flush_profile() override { ++_profile_flushes; } + bool crossed_page_since_last_batch() override { + ++_page_crossing_checks; + return _crossed_page; + } + + void set_crossed_page(bool crossed_page) { _crossed_page = crossed_page; } + int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } + int profile_flushes() const { return _profile_flushes; } + int page_crossing_checks() const { return _page_crossing_checks; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; -}; - -class ScriptedNestedReader final : public ParquetColumnReader { -public: - ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, - std::vector def_levels, std::vector rep_levels, - bool has_repeated_child = false, bool build_nulls = false) - : ParquetColumnReader(schema, std::move(type)), - _def_levels(std::move(def_levels)), - _rep_levels(std::move(rep_levels)), - _has_repeated_child(has_repeated_child), - _build_nulls(build_nulls) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - _build_lengths.push_back(length_upper_bound); - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("invalid scripted nested build arguments"); - } - for (int64_t row = 0; row < length_upper_bound; ++row) { - insert_value(column, _next_value++, _build_nulls); - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, - static_cast(_def_levels.size()))); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return _has_repeated_child; } - - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - -private: - static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - if (is_null) { - nullable_column->insert_default(); - return; - } - assert_cast(nullable_column->get_nested_column()).insert_value(value); - nullable_column->get_null_map_data().push_back(0); - return; - } - assert_cast(*column).insert_value(value); - } - - std::vector _def_levels; - std::vector _rep_levels; - bool _has_repeated_child = false; - bool _build_nulls = false; - int64_t _next_value = 0; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; -}; - -class ChunkedNestedLeafReader final : public ParquetColumnReader { -public: - ChunkedNestedLeafReader() - : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), - std::make_shared()) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(values_read != nullptr); - _initial_column_sizes.push_back(column->size()); - _build_lengths.push_back(length_upper_bound); - if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { - auto& values = assert_cast(nullable->get_nested_column()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values.insert_value(row); - nullable->get_null_map_data().push_back(0); - } - } else { - auto* values = assert_cast(column.get()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values->insert_value(row); - } - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return true; } - - const std::vector& load_lengths() const { return _load_lengths; } - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - const std::vector& initial_column_sizes() const { return _initial_column_sizes; } - -private: - std::vector _def_levels; - std::vector _rep_levels; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; - std::vector _initial_column_sizes; -}; - -} // namespace - -struct ScalarColumnReaderTestAccess { - static void set_nested_batch(ScalarColumnReader* reader, - std::unique_ptr batch) { - reader->_nested_batch = std::move(batch); - } - - static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { - return reader.page_filtered_rows_to_skip(rows); - } - - static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { - reader->_row_group_rows_read = rows; - } - - static Status append_dictionary_filtered_values( - const ScalarColumnReader& reader, - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { - return reader.append_dictionary_filtered_values(chunks, dictionary_filter, column, - row_filter, matched_rows, used_filter); - } -}; - -namespace { - -ParquetColumnSchema string_schema(std::string name = "string") { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.type = std::make_shared(); - schema.type_descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - schema.type_descriptor.doris_type = schema.type; - return schema; -} - -std::shared_ptr<::arrow::Array> dictionary_array(const std::vector& indices, - const std::vector& values) { - ::arrow::Int8Builder index_builder; - EXPECT_TRUE(index_builder.AppendValues(indices).ok()); - auto index_result = index_builder.Finish(); - EXPECT_TRUE(index_result.ok()) << index_result.status(); - - ::arrow::StringBuilder dictionary_builder; - EXPECT_TRUE(dictionary_builder.AppendValues(values).ok()); - auto dictionary_result = dictionary_builder.Finish(); - EXPECT_TRUE(dictionary_result.ok()) << dictionary_result.status(); - - auto result = ::arrow::DictionaryArray::FromArrays( - ::arrow::dictionary(::arrow::int8(), ::arrow::utf8()), *index_result, - *dictionary_result); - EXPECT_TRUE(result.ok()) << result.status(); - return *result; -} - -std::unique_ptr make_scripted_scalar_reader( - ParquetColumnSchema schema, std::unique_ptr batch) { - auto reader = std::make_unique(schema, nullptr); - ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); - return reader; -} - -std::unique_ptr scalar_batch(std::vector def_levels, - std::vector rep_levels, - std::vector value_indices, - std::vector values) { - auto batch = std::make_unique(); - batch->levels_written = static_cast(def_levels.size()); - batch->def_levels = std::move(def_levels); - batch->rep_levels = std::move(rep_levels); - batch->value_indices = std::move(value_indices); - auto column = ColumnInt64::create(); - for (const auto value : values) { - column->insert_value(value); - } - batch->values_column = std::move(column); - return batch; -} - -class DefaultOnlyReader final : public ParquetColumnReader { -public: - DefaultOnlyReader() - : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { - } - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } + int _profile_flushes = 0; + bool _crossed_page = false; + int _page_crossing_checks = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -480,21 +101,6 @@ GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { return location; } -TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { - ScalarColumnReader reader(string_schema("dictionary_value"), nullptr); - MutableColumnPtr column = ColumnString::create(); - IColumn::Filter row_filter; - int64_t matched_rows = 0; - bool used_filter = false; - const std::vector> chunks = { - dictionary_array({0, 1}, {"keep", "out-of-range"})}; - - const auto status = ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1}, column, &row_filter, &matched_rows, &used_filter); - EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; - EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); -} - } // namespace TEST(SelectionVectorTest, IdentitySelectionToRanges) { @@ -520,6 +126,21 @@ TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); } +TEST(SelectionVectorTest, OutputRangesReuseCapacity) { + SelectionVector::Index indices[] = {1, 2, 5}; + SelectionVector selection(indices, std::size(indices)); + std::vector ranges; + ranges.reserve(8); + const auto retained_capacity = ranges.capacity(); + + selection_to_ranges(selection, std::size(indices), &ranges); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges.capacity(), retained_capacity); + selection_to_ranges(selection, 1, &ranges); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges.capacity(), retained_capacity); +} + TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); @@ -534,6 +155,28 @@ TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { EXPECT_FALSE(selection.verify(2, 3).ok()); } +TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { + SelectionVector selection(4); + selection.set_index(0, 1); + selection.set_index(1, 3); + const uint8_t* first_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &first_filter).ok()); + ASSERT_NE(first_filter, nullptr); + EXPECT_EQ(std::vector(first_filter, first_filter + 4), + std::vector({0, 1, 0, 1})); + + const uint8_t* reused_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &reused_filter).ok()); + EXPECT_EQ(reused_filter, first_filter); + + selection.set_index(1, 2); + const uint8_t* updated_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &updated_filter).ok()); + EXPECT_EQ(updated_filter, first_filter); + EXPECT_EQ(std::vector(updated_filter, updated_filter + 4), + std::vector({0, 1, 1, 0})); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -559,571 +202,67 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { SelectionVector selection; auto column = std::make_shared()->create_column(); ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); - EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(column->empty()); EXPECT_EQ(reader.cursor(), 4); EXPECT_TRUE(reader.read_lengths().empty()); EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } -TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { - DefaultOnlyReader base_reader; - EXPECT_FALSE(base_reader.skip(1).ok()); - EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); - - auto column = std::make_shared()->create_column(); - int64_t values_read = 0; - EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); - - int64_t values_consumed = 0; - EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); -} - -TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { - auto element_reader = std::make_unique(); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - ASSERT_TRUE(reader.skip(8193).ok()); - EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); - EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); - EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); - EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); -} - -TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { - const std::vector def_levels {3, 3, 1, 3}; - const std::vector rep_levels {0, 1, 0, 0}; - auto key_reader = std::make_unique( - nested_int64_schema("key", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* key_reader_ptr = key_reader.get(); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { - const std::vector def_levels {2, 0, 2}; - const std::vector rep_levels {0, 0, 0}; - auto shape_reader = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* shape_reader_ptr = shape_reader.get(); - auto child_reader = std::make_unique( - nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* child_reader_ptr = child_reader.get(); - std::vector> children; - children.push_back(std::move(shape_reader)); - children.push_back(std::move(child_reader)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {-1, 0}); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { - auto leaf_reader = std::make_unique( - nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), - std::vector {1, 1}, std::vector {0, 0}); - auto* leaf_reader_ptr = leaf_reader.get(); - const auto inner_type = std::make_shared(std::make_shared()); - auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), - inner_type, std::move(leaf_reader)); - const auto outer_type = std::make_shared(inner_type); - ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); - - ASSERT_TRUE(reader.skip(2).ok()); - EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); - EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); - EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { - ColumnArray::Offsets64 offsets; - append_offsets(offsets, {3, 0, 2}); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 3); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 5); - append_offsets(offsets, {1, 4}); - ASSERT_EQ(offsets.size(), 5); - EXPECT_EQ(offsets[3], 6); - EXPECT_EQ(offsets[4], 10); - - const NullMap parent_nulls = {0, 1, 0}; - append_parent_nulls(nullptr, parent_nulls); - NullMap dst = {1}; - append_parent_nulls(&dst, parent_nulls); - EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); -} - -TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { - ParquetPageSkipPlan page_skip_plan; - page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; - - auto schema = nested_int64_schema("page_filtered", 0, 0); - ScalarColumnReader reader(schema, nullptr, &page_skip_plan); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); - - ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); -} - -TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { - auto repeated_child = std::make_unique( - nested_int64_schema("repeated_shape", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, - std::vector {0, 0, 0, 0}, true); - auto* repeated_child_ptr = repeated_child.get(); - auto scalar_child = make_scripted_scalar_reader( - nested_int64_schema("scalar_child", 1, 2), - scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); - auto* scalar_child_ptr = scalar_child.get(); - - std::vector> children; - children.push_back(std::move(repeated_child)); - children.push_back(std::move(scalar_child)); - StructColumnReader reader(nested_struct_schema(), - make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})), - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); - EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); -} - -TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { - auto first_child = std::make_unique( - nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 0}, std::vector {0, 0}, true); - auto second_child = std::make_unique( - nested_int64_schema("second", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2}, - std::vector {0, 0}, true); - - std::vector> children; - children.push_back(std::move(first_child)); - children.push_back(std::move(second_child)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(rows_read, 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); - - ParquetColumnSchema map_schema = nested_map_schema(); - map_schema.nullable_definition_level = 2; - map_schema.definition_level = 3; - map_schema.repeated_ancestor_definition_level = 0; - auto key_reader = std::make_unique( - nested_int64_schema("key", 3, 3, 1, 0), - make_nullable(std::make_shared()), - std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); - auto value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), - scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, - {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); - auto map_reader = std::make_unique( - map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); - - std::vector> children; - children.push_back(std::move(shape_child)); - children.push_back(std::move(map_reader)); - auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, - Strings {"partitionValues"})); - StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(6, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 6); - - const auto& nullable_struct = assert_cast(*column); - ASSERT_EQ(nullable_struct.size(), 6); - EXPECT_FALSE(nullable_struct.is_null_at(0)); - EXPECT_FALSE(nullable_struct.is_null_at(1)); - EXPECT_TRUE(nullable_struct.is_null_at(2)); - EXPECT_TRUE(nullable_struct.is_null_at(3)); - EXPECT_FALSE(nullable_struct.is_null_at(4)); - EXPECT_FALSE(nullable_struct.is_null_at(5)); - - const auto& struct_column = - assert_cast(nullable_struct.get_nested_column()); - const auto& map_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_nullable.size(), 6); - EXPECT_FALSE(map_nullable.is_null_at(0)); - EXPECT_FALSE(map_nullable.is_null_at(1)); - EXPECT_TRUE(map_nullable.is_null_at(2)); - EXPECT_TRUE(map_nullable.is_null_at(3)); - EXPECT_FALSE(map_nullable.is_null_at(4)); - EXPECT_FALSE(map_nullable.is_null_at(5)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 6); - EXPECT_EQ(map_column.get_offsets()[0], 1); - EXPECT_EQ(map_column.get_offsets()[1], 2); - EXPECT_EQ(map_column.get_offsets()[2], 2); - EXPECT_EQ(map_column.get_offsets()[3], 2); - EXPECT_EQ(map_column.get_offsets()[4], 3); - EXPECT_EQ(map_column.get_offsets()[5], 4); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 0, 2}, std::vector {0, 0, 0}); - - auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); - id_batch->value_slot_definition_level = 3; - auto id_reader = - make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); - - ParquetColumnSchema inner_schema; - inner_schema.local_id = 0; - inner_schema.name = "stats_parsed"; - inner_schema.kind = ParquetColumnSchemaKind::STRUCT; - inner_schema.nullable_definition_level = 2; - inner_schema.definition_level = 3; - inner_schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); - - std::vector> inner_children; - inner_children.push_back(std::move(id_reader)); - auto inner_reader = std::make_unique( - inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); - - std::vector> outer_children; - outer_children.push_back(std::move(shape_child)); - outer_children.push_back(std::move(inner_reader)); - auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, - Strings {"stats_parsed"})); - StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), - {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& outer_nullable = assert_cast(*column); - ASSERT_EQ(outer_nullable.size(), 3); - EXPECT_FALSE(outer_nullable.is_null_at(0)); - EXPECT_TRUE(outer_nullable.is_null_at(1)); - EXPECT_FALSE(outer_nullable.is_null_at(2)); +TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBoundary) { + ParquetScanScheduler scheduler; + auto reader = std::make_unique(); + auto* reader_ptr = reader.get(); + scheduler._current_predicate_columns.emplace(0, std::move(reader)); - const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); - const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); - ASSERT_EQ(inner_nullable.size(), 3); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_TRUE(inner_nullable.is_null_at(1)); - EXPECT_FALSE(inner_nullable.is_null_at(2)); - - const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); - const auto& id_nullable = assert_cast(inner_struct.get_column(0)); - const auto& id_values = assert_cast(id_nullable.get_nested_column()); - EXPECT_EQ(id_values.get_element(0), 10); - EXPECT_EQ(id_values.get_element(2), 20); + scheduler.flush_current_reader_profiles(); + EXPECT_EQ(reader_ptr->profile_flushes(), 1); } -TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { - auto element_reader = std::make_unique( - nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), - std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); +TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { + ParquetScanScheduler scheduler; + auto predicate_reader = std::make_unique(); + auto* predicate_ptr = predicate_reader.get(); + predicate_ptr->set_crossed_page(true); + scheduler._current_predicate_columns.emplace(0, std::move(predicate_reader)); - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + lazy_ptr->set_crossed_page(true); + scheduler._current_non_predicate_columns.emplace(1, std::move(lazy_reader)); - const auto& array_column = assert_cast(*column); - ASSERT_EQ(array_column.get_offsets().size(), 3); - EXPECT_EQ(array_column.get_offsets()[0], 0); - EXPECT_EQ(array_column.get_offsets()[1], 2); - EXPECT_EQ(array_column.get_offsets()[2], 2); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); + // Both readers are sampled even after the OR becomes true so their next batch starts cleanly. + EXPECT_TRUE(scheduler.finish_current_reader_batch_profiles()); + EXPECT_EQ(predicate_ptr->page_crossing_checks(), 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } -TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { - auto element_reader = - std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), - make_nullable(std::make_shared()), - std::vector {1, 5, 5, 5, 2, 5, 2, 0}, - std::vector {0, 0, 2, 1, 0, 1, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - - const auto inner_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()))); - auto inner_reader = std::make_unique( - nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, - 2), - inner_type, std::move(element_reader)); - auto outer_type = make_nullable(std::make_shared(inner_type)); - ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, - std::move(inner_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 5); - EXPECT_EQ(outer_offsets[3], 5); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 5); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_TRUE(inner_nullable.is_null_at(4)); +TEST(ParquetColumnReaderControlTest, PendingOutputDrainsBeforePageCrossingSample) { + ParquetScanScheduler scheduler; + scheduler._batch_size = 1; + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + scheduler._current_non_predicate_columns.emplace(format::LocalColumnId(0), + std::move(lazy_reader)); + scheduler._pending_predicate_batch_rows = 2; + scheduler._pending_predicate_selection = {0, 1}; - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 5); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 3); - EXPECT_EQ(inner_offsets[2], 3); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 4); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); -} - -TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + Block block; + auto type = std::make_shared(); + block.insert({type->create_column(), type, "mock"}); + size_t rows = 0; - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 1); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 0); + lazy_ptr->set_crossed_page(true); - const auto& nullable_map = assert_cast(*column); - EXPECT_FALSE(nullable_map.is_null_at(0)); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); -} - -TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 4, 4, 2, 4), - make_nullable(std::make_shared()), std::vector {1, 4}, - std::vector {0, 0}); - auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), - scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); - - const auto map_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()), - make_nullable(std::make_shared()))); - auto map_reader = std::make_unique( - nested_map_schema(make_nullable(std::make_shared())), map_type, - std::move(key_reader), std::move(value_reader)); - auto outer_type = make_nullable(std::make_shared(map_type)); - ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, - std::move(map_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 2); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 2); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 1); - - const auto& map_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(map_nullable.size(), 1); - EXPECT_FALSE(map_nullable.is_null_at(0)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 1); - - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 1); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_EQ(value_data.get_element(0), 100); -} - -TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}, false, true); - auto value_reader = std::make_unique( - nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}); - MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(value_reader)); - auto column = null_key_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = null_key_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); - - auto aligned_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(aligned_key_reader), - std::move(misaligned_value_reader)); - column = misaligned_reader.type()->create_column(); - status = misaligned_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { - auto null_key_reader = - make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), - scalar_batch({2}, {0}, {-1}, std::vector {})); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {2}, - std::vector {0}); - MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, - std::move(null_key_reader), std::move(value_reader)); - int64_t values_consumed = 0; - auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null"), std::string::npos); - - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(misaligned_value_reader)); - status = misaligned_reader.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto scalar_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); - MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(scalar_value_reader)); - auto column = scalar_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = scalar_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_map = assert_cast(*column); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 2); - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 2); - EXPECT_EQ(value_data.get_element(0), 100); - EXPECT_EQ(value_data.get_element(1), 200); - - auto complex_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto complex_value_reader = std::make_unique( - nested_int64_schema("complex_value", 2, 3, 1), - make_nullable(std::make_shared()), std::vector {3, 3}, - std::vector {0, 1}); - auto* complex_value_reader_ptr = complex_value_reader.get(); - MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, - std::move(complex_key_reader), std::move(complex_value_reader)); - column = complex_reader.type()->create_column(); - status = complex_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); + block.get_by_position(0).column = type->create_column(); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_TRUE(scheduler._pending_predicate_selection.empty()); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { @@ -1192,79 +331,4 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { EXPECT_FALSE(reader.read(1, column, nullptr).ok()); } -TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { - ParquetColumnSchema schema = int64_schema("bad_leaf"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.leaf_column_id = 3; - schema.type_descriptor.physical_type = ::parquet::Type::INT64; - schema.type_descriptor.doris_type = schema.type; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsProjectedUnsupportedLogicalType) { - ParquetColumnSchema schema = int64_schema("unsupported_time"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.type_descriptor.unsupported_reason = - "Parquet TIME with isAdjustedToUTC=true is not supported"; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(schema.type_descriptor.unsupported_reason), - std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { - auto schema = struct_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto empty_projection = format::LocalColumnIndex::partial_local(0); - status = factory.create(schema, &empty_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { - auto schema = list_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto projection = format::LocalColumnIndex::partial_local(0); - const auto status = factory.create(schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { - auto schema = map_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto key_only_projection = format::LocalColumnIndex::partial_local(0); - key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); - status = factory.create(schema, &key_only_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); -} - } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 7402595fc37d06..adb3b4b383c9c8 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -35,10 +36,15 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -58,6 +64,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -66,6 +73,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -142,6 +150,86 @@ class Int32GreaterThanExpr final : public VExpr { const std::string _expr_name = "Int32GreaterThanExpr"; }; +class Int32DictionaryEqualsExpr final : public VExpr { +public: + Int32DictionaryEqualsExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto expected = Field::create_field(_value); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value == expected; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "Int32DictionaryEqualsExpr"; +}; + +class DictionaryAcceptAllExpr final : public VExpr { +public: + explicit DictionaryAcceptAllExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + return ctx.slot(_column_id) == nullptr ? ZoneMapFilterResult::kUnsupported + : ZoneMapFilterResult::kMayMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::string _expr_name = "DictionaryAcceptAllExpr"; +}; + class Int32SumGreaterThanExpr final : public VExpr { public: Int32SumGreaterThanExpr(int left_column_id, int right_column_id, int32_t value) @@ -376,6 +464,21 @@ VExprContextSPtr create_int32_greater_than_conjunct(int column_id, int32_t value return ctx; } +VExprContextSPtr create_int32_dictionary_equals_conjunct(int column_id, int32_t value) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, value)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_dictionary_accept_all_conjunct(int column_id) { + auto ctx = VExprContext::create_shared(std::make_shared(column_id)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + VExprContextSPtr create_int32_sum_greater_than_conjunct(int left_column_id, int right_column_id, int32_t value) { auto ctx = VExprContext::create_shared( @@ -458,6 +561,39 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_nullable_int32_array( + const std::vector>& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(value.has_value() ? builder.Append(*value).ok() : builder.AppendNull().ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_int64_array(const std::vector& values) { + arrow::Int64Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_float_array(const std::vector& values) { + arrow::FloatBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_double_array(const std::vector& values) { + arrow::DoubleBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -466,6 +602,14 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_timestamp_array(const std::shared_ptr& type, const std::vector& values) { arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); @@ -475,6 +619,27 @@ std::shared_ptr build_timestamp_array(const std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + const int32_t byte_width = + std::static_pointer_cast(type)->byte_width(); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_struct_array(const std::vector& ids, const std::vector& names) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), @@ -517,6 +682,41 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_unannotated_binary_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("raw_bytes", arrow::binary(), false)}); + auto table = arrow::Table::Make(schema, {build_binary_array({"否", "是", "测试"})}); + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + builder.build())); +} + +void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { + auto decimal_type = arrow::decimal128(38, 6); + auto fixed_type = arrow::fixed_size_binary(4); + auto schema = arrow::schema({arrow::field("decimal_value", decimal_type, false), + arrow::field("fixed_value", fixed_type, false)}); + auto table = arrow::Table::Make( + schema, + {build_decimal_array(decimal_type, {1234567, -1, 0, -987654321, 42}), + build_fixed_binary_array(fixed_type, {"ABCD", std::string("\0x\0y", 4), "wxyz", "1234", + std::string("\xff\x00\x7f\x80", 4)})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + std::shared_ptr build_nullable_int_string_map_array() { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); @@ -686,6 +886,27 @@ void write_nullable_string_struct_parquet_file(const std::string& file_path) { ROW_COUNT, builder.build())); } +void write_nullable_complex_parquet_file(const std::string& file_path) { + auto map_array = build_nullable_int_string_map_array(); + auto list_array = build_nullable_string_list_array(); + auto struct_array = build_nullable_string_struct_array(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("m", map_array->type(), true), + arrow::field("a", list_array->type(), true), + arrow::field("s", struct_array->type(), true)}), + {map_array, list_array, struct_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { auto scalar_first = build_nullable_struct_with_list_array(false); auto list_first = build_nullable_struct_with_list_array(true); @@ -706,6 +927,34 @@ void write_nullable_struct_with_list_parquet_file(const std::string& file_path) ROW_COUNT, builder.build())); } +void write_nested_complex_under_struct_parquet_file(const std::string& file_path) { + auto nested_struct = build_nullable_string_struct_array(); + auto nested_array = build_nullable_string_list_array(); + auto nested_map = build_nullable_int_string_map_array(); + auto marker = build_int32_array({10, 20, 30, 40, 50}); + auto struct_field = arrow::field("nested_struct", nested_struct->type(), true); + auto array_field = arrow::field("nested_array", nested_array->type(), true); + auto map_field = arrow::field("nested_map", nested_map->type(), true); + auto marker_field = arrow::field("marker", arrow::int32(), false); + auto outer_result = + arrow::StructArray::Make({nested_struct, nested_array, nested_map, marker}, + {struct_field, array_field, map_field, marker_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer = *outer_result; + auto table = arrow::Table::Make(arrow::schema({arrow::field("outer", outer->type(), false)}), + {outer}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_int96_timestamp_parquet_file(const std::string& file_path) { auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); auto array = @@ -792,14 +1041,21 @@ void write_struct_filter_parquet_file(const std::string& file_path) { builder.build())); } -void write_dictionary_filter_parquet_file(const std::string& file_path) { +void write_dictionary_filter_parquet_file( + const std::string& file_path, + ::parquet::Compression::type compression = ::parquet::Compression::UNCOMPRESSED) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + const std::vector values = + compression == ::parquet::Compression::UNCOMPRESSED + ? std::vector {"aa", "az", "lm", "lz", "za", "zz"} + : std::vector {std::string(4096, 'a'), std::string(4096, 'b'), + std::string(4096, 'c'), std::string(4096, 'd'), + std::string(4096, 'e'), std::string(4096, 'f')}; + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_string_array(values)}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -808,7 +1064,7 @@ void write_dictionary_filter_parquet_file(const std::string& file_path) { ::parquet::WriterProperties::Builder builder; builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.compression(compression); builder.enable_dictionary("value"); builder.disable_dictionary("id"); builder.disable_statistics(); @@ -840,16 +1096,14 @@ void write_single_row_group_dictionary_filter_parquet_file(const std::string& fi builder.build())); } -void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { +void write_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), - arrow::field("value", arrow::utf8(), false), - arrow::field("payload", arrow::int32(), false), + arrow::field("value", arrow::int32(), true), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), - build_int32_array({10, 20, 30, 40, 50, 60})}); + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_nullable_int32_array({10, 20, std::nullopt, 20, 40, 20})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -861,46 +1115,62 @@ void write_dictionary_filter_with_trailing_column_parquet_file(const std::string builder.compression(::parquet::Compression::UNCOMPRESSED); builder.disable_dictionary("id"); builder.enable_dictionary("value"); - builder.disable_dictionary("payload"); builder.disable_statistics(); PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); +void write_all_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { + auto fixed_type = arrow::fixed_size_binary(4); + auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO); auto schema = arrow::schema({ - arrow::field("s", struct_type, false), + arrow::field("id", arrow::int32(), false), + arrow::field("int64_value", arrow::int64(), false), + arrow::field("float_value", arrow::float32(), false), + arrow::field("double_value", arrow::float64(), false), + arrow::field("fixed_value", fixed_type, false), + arrow::field("int96_value", timestamp_type, false), }); auto table = arrow::Table::Make( - schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); + schema, + {build_int32_array({1, 2, 3, 4, 5, 6}), build_int64_array({10, 20, 10, 30, 20, 10}), + build_float_array({1.5F, 2.5F, 1.5F, 3.5F, 2.5F, 1.5F}), + build_double_array({10.25, 20.25, 10.25, 30.25, 20.25, 10.25}), + build_fixed_binary_array(fixed_type, {"AAAA", "BBBB", "AAAA", "CCCC", "BBBB", "AAAA"}), + build_timestamp_array(timestamp_type, {1000, 2000, 1000, 3000, 2000, 1000})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("s.name"); - builder.disable_dictionary("s.identifier.field_id"); - builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, - builder.build())); + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + writer_builder.disable_dictionary("id"); + writer_builder.enable_dictionary("int64_value"); + writer_builder.enable_dictionary("float_value"); + writer_builder.enable_dictionary("double_value"); + writer_builder.enable_dictionary("fixed_value"); + writer_builder.enable_dictionary("int96_value"); + writer_builder.disable_statistics(); + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_force_write_int96_timestamps(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, + writer_builder.build(), + arrow_builder.build())); } -void write_dictionary_edge_parquet_file(const std::string& file_path) { +void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), + arrow::field("payload", arrow::int32(), false), }); - auto table = arrow::Table::Make( - schema, - {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), - build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), + build_int32_array({10, 20, 30, 40, 50, 60})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -910,52 +1180,23 @@ void write_dictionary_edge_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("value"); builder.disable_dictionary("id"); + builder.enable_dictionary("value"); + builder.disable_dictionary("payload"); builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - std::vector names; - names.reserve(ids.size()); - for (const auto id : ids) { - names.push_back("name-" + std::to_string(id)); - } - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - -void write_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); +void write_dictionary_edge_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + auto table = arrow::Table::Make( + schema, + {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), + build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -965,12 +1206,11 @@ void write_page_index_filter_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); } void write_page_index_filter_pair_parquet_file(const std::string& file_path) { @@ -1136,7 +1376,8 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false, + std::string fs_name = {}, int64_t mtime = 0) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1145,9 +1386,11 @@ class NewParquetReaderTest : public testing::Test { file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; + file_description->fs_name = std::move(fs_name); + file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); } std::filesystem::path _test_dir; @@ -1172,6 +1415,39 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, RawByteArrayMappingFollowsV2ScanOption) { + write_unannotated_binary_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto string_reader = create_reader(); + const auto string_init_status = string_reader->init(&state); + ASSERT_TRUE(string_init_status.ok()) << string_init_status; + std::vector string_schema; + ASSERT_TRUE(string_reader->get_schema(&string_schema).ok()); + ASSERT_EQ(string_schema.size(), 1); + EXPECT_EQ(remove_nullable(string_schema[0].type)->get_primitive_type(), TYPE_STRING); + + RuntimeProfile binary_profile("raw_binary_mapping_profile"); + auto binary_reader = + create_reader(0, -1, &binary_profile, false, nullptr, std::nullopt, false, true); + const auto binary_init_status = binary_reader->init(&state); + ASSERT_TRUE(binary_init_status.ok()) << binary_init_status; + std::vector binary_schema; + ASSERT_TRUE(binary_reader->get_schema(&binary_schema).ok()); + ASSERT_EQ(binary_schema.size(), 1); + // The explicit VARBINARY mapping must remain available for scans whose table contract asks for it. + EXPECT_EQ(remove_nullable(binary_schema[0].type)->get_primitive_type(), TYPE_VARBINARY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_string_in_conjunct(0, {"是"})); + ASSERT_TRUE(binary_reader->open(request).ok()); + // A table-side STRING comparison may be rewritten through this VARBINARY file slot. Native + // dictionary pruning must leave the row group available for the mapping expression. + EXPECT_EQ(binary_profile.get_counter("RowGroupsFilteredByDictionary")->value(), 0); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1203,6 +1479,10 @@ TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { EXPECT_EQ(result.count, 4); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); + ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); + ASSERT_NE(profile.get_counter("ParsePageHeaderNum"), nullptr); + EXPECT_GT(profile.get_counter("ParsePageHeaderNum")->value(), 0); } TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { @@ -1271,6 +1551,238 @@ TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundari } } +TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchChanges) { + write_nullable_complex_parquet_file(_file_path); + RuntimeProfile profile("native_complex_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + int batch = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + if (++batch == 1) { + // Adaptive sizing changes only the logical row cap. The persistent native readers and + // their level/string scratch must continue from the same page cursors. + reader->set_batch_size(3); + } + } + + const auto& nullable_map = assert_cast(*output[0]); + ASSERT_EQ(nullable_map.size(), ROW_COUNT); + EXPECT_FALSE(nullable_map.is_null_at(0)); + EXPECT_TRUE(nullable_map.is_null_at(1)); + EXPECT_FALSE(nullable_map.is_null_at(2)); + const auto& map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ(map.get_offsets(), ColumnArray::Offsets64({1, 1, 1, 2, 3})); + const auto& map_keys = assert_cast(map.get_keys()); + const auto& key_values = assert_cast(map_keys.get_nested_column()); + ASSERT_EQ(key_values.size(), 3); + EXPECT_EQ(key_values.get_element(0), 10); + EXPECT_EQ(key_values.get_element(2), 30); + const auto& map_values = assert_cast(map.get_values()); + const auto& value_strings = assert_cast(map_values.get_nested_column()); + EXPECT_EQ(value_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(value_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(map_values.is_null_at(2)); + + const auto& nullable_array = assert_cast(*output[1]); + ASSERT_EQ(nullable_array.size(), ROW_COUNT); + EXPECT_TRUE(nullable_array.is_null_at(1)); + const auto& array = assert_cast(nullable_array.get_nested_column()); + EXPECT_EQ(array.get_offsets(), ColumnArray::Offsets64({2, 2, 2, 3, 4})); + const auto& array_values = assert_cast(array.get_data()); + const auto& array_strings = assert_cast(array_values.get_nested_column()); + EXPECT_EQ(array_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(array_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(array_values.is_null_at(2)); + EXPECT_EQ(array_strings.get_data_at(3).size, 4096); + + const auto& nullable_struct = assert_cast(*output[2]); + ASSERT_EQ(nullable_struct.size(), ROW_COUNT); + EXPECT_TRUE(nullable_struct.is_null_at(1)); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& payload = assert_cast(struct_column.get_column(0)); + const auto& payload_strings = assert_cast(payload.get_nested_column()); + EXPECT_EQ(payload_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(payload_strings.get_data_at(2).size, 4096); + EXPECT_TRUE(payload.is_null_at(3)); + EXPECT_EQ(payload_strings.get_data_at(4).size, 4096); + const auto& ids = assert_cast(struct_column.get_column(1)); + const auto& id_values = assert_cast(ids.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 1); + EXPECT_EQ(id_values.get_element(4), 4); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); + EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); +} + +TEST_F(NewParquetReaderTest, FullComplexChildUnderPartialParentReadsItsWholeSubtree) { + write_nested_complex_under_struct_parquet_file(_file_path); + for (size_t child_index = 0; child_index < 3; ++child_index) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + ASSERT_EQ(schema[0].children.size(), 4); + auto projection = format::LocalColumnIndex::partial_local(0); + projection.children.push_back( + format::LocalColumnIndex::local(schema[0].children[child_index].file_local_id())); + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + format::ColumnDefinition projected; + ASSERT_TRUE(format::project_column_definition(schema[0], projection, &projected).ok()); + Block block; + block.insert(ColumnWithTypeAndName(projected.type->create_column(), projected.type, + projected.name)); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + const auto& outer = nullable_nested_column(block, 0); + ASSERT_EQ(outer.tuple_size(), 1); + const auto& nullable_nested = assert_cast(outer.get_column(0)); + if (child_index == 0) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + ASSERT_EQ(nested.tuple_size(), 2); + const auto& payload = assert_cast(nested.get_column(0)); + EXPECT_EQ(assert_cast(payload.get_nested_column()) + .get_data_at(0) + .to_string(), + "small"); + const auto& ids = assert_cast(nested.get_column(1)); + EXPECT_EQ(ids.get_nested_column().get_int(0), 1); + } else if (child_index == 1) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 2); + EXPECT_EQ(nested.get_data().size(), 4); + } else { + const auto& nested = assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 1); + EXPECT_EQ(nested.get_keys().size(), 3); + EXPECT_EQ(nested.get_values().size(), 3); + } + } +} + +TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + _file_path = + std::string(source_root) + "/regression-test/data/external_table_p0/tvf/comp.parquet"; + ASSERT_TRUE(std::filesystem::exists(_file_path)); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t position = 0; position < schema.size(); ++position) { + request->non_predicate_columns.push_back(field_projection(cast_set(position))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const Status status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + EXPECT_GT(total_rows, 0); +} + +TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { + write_decimal_and_fixed_binary_parquet_file(_file_path); + RuntimeProfile profile("native_decimal_fixed_binary_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + reader->set_batch_size(3); + } + + const auto& decimals = assert_cast( + assert_cast(*output[0]).get_nested_column()); + ASSERT_EQ(decimals.size(), ROW_COUNT); + EXPECT_EQ(decimals.get_element(0), Decimal128V3(1234567)); + EXPECT_EQ(decimals.get_element(1), Decimal128V3(-1)); + EXPECT_EQ(decimals.get_element(3), Decimal128V3(-987654321)); + + const auto& fixed_values = assert_cast( + assert_cast(*output[1]).get_nested_column()); + ASSERT_EQ(fixed_values.size(), ROW_COUNT); + EXPECT_EQ(fixed_values.get_data_at(0).to_string(), "ABCD"); + EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); + EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + EXPECT_EQ(profile.get_counter("ConvertTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { write_struct_filter_parquet_file(_file_path); auto reader = create_reader(); @@ -1297,6 +1809,35 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); } +TEST_F(NewParquetReaderTest, ComplexColumnDoesNotMisreportSiblingPagesAsCrossing) { + write_struct_filter_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_complex_page_fragments"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 4); + // Each STRUCT child reads one page per Row Group, but no individual leaf crosses a boundary in + // either batch. The aggregate fragment count remains useful without inflating crossing batches. + EXPECT_EQ(profile.get_counter("NativePageFragments")->value(), 8); + EXPECT_EQ(profile.get_counter("PageCrossingBatches")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { write_int96_timestamp_parquet_file(_file_path); auto reader = create_reader(0, -1, nullptr, true); @@ -1532,10 +2073,78 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr); - EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeFooterCacheIdentityIncludesFilesystemAndVersion) { + const auto first = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto other_fs = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-b", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto replaced = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 11, 0, 1024, 1024, false); + EXPECT_FALSE(first.empty()); + EXPECT_NE(first, other_fs); + EXPECT_NE(first, replaced); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotCrossFilesystems) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("native_footer_cache_nameservice_a"); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-a", 1234567); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile second_profile("native_footer_cache_nameservice_b"); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-b", 1234567); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheSkipsMutableUnknownVersion) { + EXPECT_TRUE(format::parquet::detail::build_native_file_cache_key("hdfs://nameservice-a", + "/warehouse/mutable.parquet", + 0, 0, 1024, 1024, false) + .empty()); + EXPECT_FALSE( + format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/immutable.parquet", 0, 0, 1024, 1024, true) + .empty()); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) { + _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); + write_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("native_footer_cache_mutable_first"); + auto first = create_reader(0, -1, &first_profile); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + write_parquet_file(_file_path); + RuntimeProfile second_profile("native_footer_cache_mutable_second"); + auto second = create_reader(0, -1, &second_profile); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { + constexpr size_t file_size = 256UL << 20; + constexpr size_t metadata_limit = 100UL << 20; + const auto status = format::parquet::detail::validate_native_footer_size( + static_cast(metadata_limit + 1), file_size, metadata_limit); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("metadata limit"), std::string::npos); +} + TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string(); write_parquet_file(_file_path); @@ -1645,24 +2254,42 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterHitCache"), nullptr); ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + TRuntimeProfileTree profile_tree; + profile.to_thrift(&profile_tree); + ASSERT_FALSE(profile_tree.nodes.empty()); + const auto parquet_children = + profile_tree.nodes.front().child_counters_map.find("ParquetReader"); + ASSERT_NE(parquet_children, profile_tree.nodes.front().child_counters_map.end()); + EXPECT_TRUE(parquet_children->second.contains("RowsFilteredByConjunct")); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + + profile.get_counter("FileFooterHitCache")->value(), + 1); rows = 0; eof = false; @@ -1777,14 +2404,61 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 0); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } +TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTotalTime) { + RuntimeProfile profile("new_parquet_reader_hierarchy_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + + auto* parquet_total = profile.get_counter("ParquetReader"); + ASSERT_NE(parquet_total, nullptr); + EXPECT_GT(parquet_total->value(), 0); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); + EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); + ASSERT_TRUE(children.contains("FileScannerV2")); + EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); + ASSERT_TRUE(children.contains("TableReader")); + EXPECT_TRUE(children.at("TableReader").contains("FileReader")); + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredBytes")); + EXPECT_TRUE(children.at("ParquetReader").contains("FileNum")); +} + TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -2010,41 +2684,6 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto value_chunk = row_group->ColumnChunk(1); - ASSERT_NE(value_chunk, nullptr); - ASSERT_TRUE(value_chunk->has_dictionary_page()); - ASSERT_TRUE(value_chunk->statistics() == nullptr || - !value_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 2); - - format::FileScanRequest plan_request; - plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - plan_request, scan_range, false, &plan) - .ok()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); - auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -2080,6 +2719,55 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { EXPECT_EQ(values, std::vector({"lm"})); } +TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProfile) { + const double old_cache_threshold = config::parquet_page_cache_decompress_threshold; + config::parquet_page_cache_decompress_threshold = 100.0; + Defer restore_cache_threshold { + [&] { config::parquet_page_cache_decompress_threshold = old_cache_threshold; }}; + write_dictionary_filter_parquet_file(_file_path, ::parquet::Compression::SNAPPY); + + auto open_pruned_reader = [&](RuntimeProfile* profile) { + auto reader = create_reader(0, -1, profile, false, nullptr, std::nullopt, true); + TQueryOptions query_options; + query_options.__set_enable_parquet_file_page_cache(true); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); + use_schema_order_positions(request.get(), schema); + RETURN_IF_ERROR(reader->open(request)); + EXPECT_EQ(profile->get_counter("RowGroupsFilteredByDictionary")->value(), 0); + EXPECT_EQ(profile->get_counter("PageReadCount")->value(), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + // Expensive dictionary probes are current-row-group work now, so advance the scheduler + // once before inspecting page-cache and pruning counters. + return reader->get_block(&block, &rows, &eof); + }; + + RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); + ASSERT_TRUE(open_pruned_reader(&cold_profile).ok()); + for (const auto* counter_name : + {"RowGroupsFilteredByDictionary", "PageReadCount", "PageCacheWriteCount", + "ParsePageHeaderNum", "DecompressCount", "DecodeDictTime"}) { + ASSERT_NE(cold_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(cold_profile.get_counter(counter_name)->value(), 0) << counter_name; + } + + RuntimeProfile warm_profile("dictionary_pruning_warm_profile"); + ASSERT_TRUE(open_pruned_reader(&warm_profile).ok()); + for (const auto* counter_name : {"RowGroupsFilteredByDictionary", "PageReadCount", + "PageCacheHitCount", "ParsePageHeaderNum", "DecodeDictTime"}) { + ASSERT_NE(warm_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(warm_profile.get_counter(counter_name)->value(), 0) << counter_name; + } +} + TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { write_single_row_group_dictionary_filter_parquet_file(_file_path); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); @@ -2135,7 +2823,102 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); } -TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { +TEST_F(NewParquetReaderTest, FixedWidthDictionaryPredicateFiltersRowsByDictionaryId) { + write_fixed_width_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_int32_dictionary_equals_conjunct(1, 20)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({2, 4, 6})); + EXPECT_EQ(values, std::vector({20, 20, 20})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 3); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, AllFixedWidthDictionaryTypesDecodeThroughDictionaryIds) { + write_all_fixed_width_dictionary_filter_parquet_file(_file_path); + + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto row_group = parquet_file_reader->metadata()->RowGroup(0); + ASSERT_NE(row_group, nullptr); + ASSERT_EQ(row_group->num_columns(), 6); + const std::array<::parquet::Type::type, 5> expected_types { + ::parquet::Type::INT64, ::parquet::Type::FLOAT, ::parquet::Type::DOUBLE, + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, ::parquet::Type::INT96}; + for (int column = 1; column < row_group->num_columns(); ++column) { + ASSERT_TRUE(row_group->ColumnChunk(column)->has_dictionary_page()) << column; + EXPECT_EQ(row_group->ColumnChunk(column)->type(), expected_types[column - 1]) << column; + } + + RuntimeProfile profile("new_parquet_reader_all_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 6); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + for (int column = 1; column < 6; ++column) { + request->predicate_columns.push_back(field_projection(column)); + request->conjuncts.push_back(create_dictionary_accept_all_conjunct(column)); + } + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + + EXPECT_EQ(total_rows, 6); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateReaderIsSharedOutsideMergeRangeReader) { write_dictionary_filter_with_trailing_column_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); @@ -2177,8 +2960,14 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + // The native dictionary probe keeps its reader and cursor for predicate data pages. Other + // projected chunks still use the row-group merge reader, so probing never duplicates the + // dictionary read or perturbs the merge range's sequential access order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { @@ -2258,7 +3047,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh // second predicate column is skipped after the selection becomes empty, which verifies the // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); - EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); + // Five dictionary ids are rejected before the residual predicate; the remaining predicate + // reader then skips all six logical rows once the selection is empty. + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 11); } TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { @@ -2346,208 +3137,10 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); } -TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { - write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { - write_struct_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { - write_nested_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto name_chunk = row_group->ColumnChunk(1); - ASSERT_NE(name_chunk, nullptr); - ASSERT_TRUE(name_chunk->has_dictionary_page()); - ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[1]->name, "name"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 6); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { - write_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - - format::FileScanRequest request; - request.predicate_columns = {field_projection(0)}; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); - auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); - EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); - EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); - ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), - skip_plan_it->second.skipped_page_compressed_sizes.size()); - int64_t skipped_compressed_bytes = 0; - for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { - if (skip_plan_it->second.should_skip_page(page_idx)) { - skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); - } - } - EXPECT_GT(skipped_compressed_bytes, 0); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { - write_nested_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, - parquet_file_reader->metadata()->RowGroup(0)->num_rows()); - EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - // Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range // gap for the non-predicate payload reader, then flushes it exactly once before materialization. The -// page skip plan advances the reader without calling Arrow SkipRecords or double-skipping row 64. +// native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or +// double-skipping row 64. TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { write_page_index_filter_pair_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_page_skip"); @@ -2571,7 +3164,8 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { bool eof = false; while (!eof) { size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -2589,7 +3183,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); @@ -2599,7 +3193,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 36cc99ebf106c0..cc1d28238cb7e1 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -23,9 +23,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -44,13 +46,16 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -58,6 +63,8 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/utils.h" +#include "util/coding.h" +#include "util/thrift_util.h" namespace doris { namespace { @@ -73,6 +80,13 @@ const ColumnInt32& int32_data_column(const IColumn& column) { return assert_cast(column); } +const ColumnInt64& int64_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + const ColumnString& string_data_column(const IColumn& column) { if (const auto* nullable_column = check_and_get_column(&column)) { return assert_cast(nullable_column->get_nested_column()); @@ -80,6 +94,64 @@ const ColumnString& string_data_column(const IColumn& column) { return assert_cast(column); } +TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssignment) { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root, leaf}); + + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + column.__set_data_page_offset(100); + column.__set_dictionary_page_offset(-1); + column.__set_total_compressed_size(20); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + metadata.__set_row_groups({row_group}); + + auto schema = std::make_unique(); + schema->local_id = 0; + schema->leaf_column_id = 0; + schema->type = std::make_shared(); + std::vector> file_schema; + file_schema.push_back(std::move(schema)); + std::vector ranges; + ASSERT_TRUE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 20); + + std::vector first_rows; + std::vector selected; + format::parquet::ParquetScanRange first_split { + .start_offset = 0, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, first_split, &first_rows, &selected) + .ok()); + EXPECT_TRUE(selected.empty()); + format::parquet::ParquetScanRange second_split { + .start_offset = 100, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, second_split, &first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({0})); + + metadata.row_groups[0].columns[0].meta_data.__set_data_page_offset(190); + EXPECT_FALSE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; @@ -195,17 +267,223 @@ class Int32PairSumExpr final : public VExpr { const std::string _expr_name = "Int32PairSumExpr"; }; +class Int32DirectGreaterExpr final : public VExpr { +public: + Int32DirectGreaterExpr(int column_id, int32_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int32_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int32_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int32_t _lower_bound; + const std::string _expr_name = "Int32DirectGreaterExpr"; +}; + +class Int64DirectGreaterExpr final : public VExpr { +public: + Int64DirectGreaterExpr(int column_id, int64_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int64_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_BIGINT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + if (value_width != sizeof(int64_t)) { + return Status::Corruption("BIGINT raw predicate received {}-byte values", value_width); + } + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int64_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int64_t _lower_bound; + const std::string _expr_name = "Int64DirectGreaterExpr"; +}; + +class AlwaysTrueSingleColumnExpr final : public VExpr { +public: + explicit AlwaysTrueSingleColumnExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + const std::string _expr_name = "AlwaysTrueSingleColumnExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); } +VExprContextSPtr create_int32_function_conjunct(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({nullable_int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(true); + auto root = VectorizedFnCall::create_shared(node); + root->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + root->add_child(VLiteral::create_shared(int_type, Field::create_field(value))); + auto context = VExprContext::create_shared(std::move(root)); + // Direct evaluation does not execute the expression, but a fallback must still fail this test + // instead of silently using an unprepared test-only context. + context->_prepared = true; + context->_opened = true; + return context; +} + +VExprContextSPtr create_int32_mod_greater_than_conjunct(int column_id) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto nullable_bool_type = make_nullable(std::make_shared()); + auto function_node = [](const std::string& name, TExprNodeType::type node_type, + TExprOpcode::type opcode, const DataTypePtr& return_type, + const std::vector& argument_types) { + TFunctionName function_name; + function_name.__set_function_name(name); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_argument_types; + for (const auto& argument_type : argument_types) { + thrift_argument_types.push_back(argument_type->to_thrift()); + } + function.__set_arg_types(thrift_argument_types); + function.__set_ret_type(return_type->to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(function); + node.__set_num_children(static_cast(argument_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + return node; + }; + + auto modulo = VectorizedFnCall::create_shared( + function_node("mod", TExprNodeType::ARITHMETIC_EXPR, TExprOpcode::MOD, + nullable_int_type, {nullable_int_type, int_type})); + modulo->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + modulo->add_child(VLiteral::create_shared(int_type, Field::create_field(-1))); + + auto greater_than = VectorizedFnCall::create_shared( + function_node("gt", TExprNodeType::BINARY_PRED, TExprOpcode::GT, nullable_bool_type, + {nullable_int_type, int_type})); + greater_than->add_child(std::move(modulo)); + greater_than->add_child(VLiteral::create_shared(int_type, Field::create_field(0))); + return VExprContext::create_shared(std::move(greater_than)); +} + VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, int32_t upper_bound) { return VExprContext::create_shared( std::make_shared(left_column_id, right_column_id, upper_bound)); } +VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_always_true_single_column_conjunct(int column_id) { + return VExprContext::create_shared(std::make_shared(column_id)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -226,6 +504,14 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -281,7 +567,8 @@ std::shared_ptr build_list_array() { void write_table(const std::string& file_path, const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary = false, - bool enable_page_index = false, bool enable_statistics = true) { + bool enable_page_index = false, bool enable_statistics = true, + std::optional<::parquet::Encoding::type> encoding = std::nullopt) { auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; @@ -295,6 +582,9 @@ void write_table(const std::string& file_path, const std::shared_ptr encoding = std::nullopt) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("score", arrow::int32(), false), }); auto table = arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_int32_array({10, 20, 30, 40, 50, 60})}); - write_table(file_path, table, row_group_size, false, false, enable_statistics); + write_table(file_path, table, row_group_size, false, false, enable_statistics, encoding); +} + +void write_uint32_pair_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("id", arrow::uint32(), false), + arrow::field("score", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_uint32_array({1, 2147483648U, 4294957294U}), + build_int32_array({10, 20, 30})}); + write_table(file_path, table, 3, false, false, false); +} + +std::vector serialize_test_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +void write_misdeclared_two_page_parquet_file(const std::string& file_path) { + const std::array first_values {1, 2}; + std::vector first_payload(sizeof(first_values)); + memcpy(first_payload.data(), first_values.data(), first_payload.size()); + tparquet::PageHeader first_header; + first_header.type = tparquet::PageType::DATA_PAGE_V2; + first_header.__set_compressed_page_size(first_payload.size()); + first_header.__set_uncompressed_page_size(first_payload.size()); + first_header.__isset.data_page_header_v2 = true; + first_header.data_page_header_v2.__set_num_values(first_values.size()); + first_header.data_page_header_v2.__set_num_nulls(0); + first_header.data_page_header_v2.__set_num_rows(first_values.size()); + first_header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + first_header.data_page_header_v2.__set_definition_levels_byte_length(0); + first_header.data_page_header_v2.__set_repetition_levels_byte_length(0); + first_header.data_page_header_v2.__set_is_compressed(false); + auto column_bytes = serialize_test_page(first_header, first_payload); + + // Bit width 1 followed by an RLE run of two dictionary ids. The payload is structurally valid + // so the raw path reaches its late-encoding guard instead of failing while loading the page. + const std::vector second_payload {1, 4, 0}; + tparquet::PageHeader second_header = first_header; + second_header.__set_compressed_page_size(second_payload.size()); + second_header.__set_uncompressed_page_size(second_payload.size()); + second_header.data_page_header_v2.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + auto second_page = serialize_test_page(second_header, second_payload); + column_bytes.insert(column_bytes.end(), second_page.begin(), second_page.end()); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("id"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + // Deliberately omit the second page's unsupported encoding to emulate untrusted footer + // metadata. The reader must reject it before attempting a typed fallback after partial cursor + // progress. + column.__set_encodings({tparquet::Encoding::PLAIN, tparquet::Encoding::RLE}); + column.__set_path_in_schema({"id"}); + column.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column.__set_num_values(4); + column.__set_total_uncompressed_size(column_bytes.size()); + column.__set_total_compressed_size(column_bytes.size()); + column.__set_data_page_offset(4); + tparquet::ColumnChunk chunk; + chunk.__set_file_offset(4); + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(column_bytes.size()); + row_group.__set_num_rows(4); + tparquet::FileMetaData metadata; + metadata.__set_version(2); + metadata.__set_schema({root, leaf}); + metadata.__set_num_rows(4); + metadata.__set_row_groups({row_group}); + + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write("PAR1", 4); + output.write(reinterpret_cast(column_bytes.data()), column_bytes.size()); + output.write(reinterpret_cast(footer.data()), footer.size()); + std::array footer_size {}; + encode_fixed32_le(footer_size.data(), cast_set(footer.size())); + output.write(reinterpret_cast(footer_size.data()), footer_size.size()); + output.write("PAR1", 4); + output.close(); + DORIS_CHECK(output.good()); +} + +void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { + std::vector ids(rows); + std::vector first(rows); + std::vector second(rows); + std::iota(ids.begin(), ids.end(), 0); + for (size_t row = 0; row < rows; ++row) { + first[row] = static_cast(row + 10); + second[row] = static_cast(row + 20); + } + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("first", arrow::int32(), false), + arrow::field("second", arrow::int32(), false)}); + auto table = arrow::Table::Make( + schema, {build_int32_array(ids), build_int32_array(first), build_int32_array(second)}); + write_table(file_path, table, rows, false, false, false); } void write_binary_minmax_parquet_file(const std::string& file_path) { @@ -392,15 +792,12 @@ void write_page_index_parquet_file(const std::string& file_path) { write_table(file_path, table, ids.size(), false, true); } -void write_page_index_pair_parquet_file(const std::string& file_path) { - std::vector ids(128); +void write_multi_row_group_page_index_parquet_file(const std::string& file_path) { + std::vector ids(384); std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - arrow::field("score", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); - write_table(file_path, table, ids.size(), false, true); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, 128, false, true); } int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { @@ -448,24 +845,6 @@ void use_schema_order_positions(format::FileScanRequest* request, } } -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - auto schema_descriptor = reader.metadata()->schema(); - EXPECT_NE(schema_descriptor, nullptr); - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - return file_schema; -} - -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - class ParquetScanTest : public testing::Test { protected: void SetUp() override { @@ -522,87 +901,68 @@ TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); } -TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); +TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map stats; + stats.emplace(0, AdaptivePredicateStats { + .cost_per_input_row_ns = 10, .survival_ratio = 0.8, .samples = 3}); + stats.emplace(1, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.1, .samples = 3}); + stats.emplace(2, AdaptivePredicateStats { + .cost_per_input_row_ns = 50, .survival_ratio = 0.5, .samples = 3}); + + const auto order = format::parquet::detail::order_adaptive_predicates({0, 1, 2}, stats); + EXPECT_EQ(order, std::vector({1, 0, 2})); + const auto prefetched = format::parquet::detail::adaptive_prefetch_prefix(order, stats, 0.25); + EXPECT_EQ(prefetched, std::vector({1})); } -TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); +TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { + using format::parquet::detail::should_sample_adaptive_predicate; + for (size_t samples = 0; samples < 8; ++samples) { + EXPECT_TRUE(should_sample_adaptive_predicate(samples, samples)); + } + EXPECT_FALSE(should_sample_adaptive_predicate(8, 9)); + EXPECT_TRUE(should_sample_adaptive_predicate(8, 16)); + EXPECT_FALSE(should_sample_adaptive_predicate(9, 17)); + EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); +} - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.row_groups[0].first_file_row, 4); - EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); - ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); -} - -TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, ThrowingNestedFunctionDisablesSelectedRowReordering) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map first_batch_stats; + first_batch_stats.emplace( + 0, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.99, .samples = 1}); + first_batch_stats.emplace( + 1, AdaptivePredicateStats { + .cost_per_input_row_ns = 1, .survival_ratio = 0.1, .samples = 1}); + EXPECT_EQ(format::parquet::detail::order_adaptive_predicates({0, 1}, first_batch_stats), + (std::vector {1, 0})); + + const auto total_comparison = create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0); + const auto throwing_comparison = create_int32_mod_greater_than_conjunct(0); + EXPECT_TRUE(total_comparison->root()->is_safe_to_execute_on_selected_rows()); + // The second-batch order above can reject MIN_INT before mod(MIN_INT, -1) runs. The nested + // partial function must therefore keep the request on full-batch, error-preserving execution. + EXPECT_FALSE(throwing_comparison->root()->is_safe_to_execute_on_selected_rows()); +} - format::FileScanRequest request; - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); +TEST(ParquetScanSmallFileTest, StagesOnlyBoundedHttpObjects) { + using format::parquet::detail::should_stage_small_http_file; + EXPECT_TRUE(should_stage_small_http_file("http://host/tiny.parquet", 512, 1024)); + EXPECT_TRUE(should_stage_small_http_file("https://host/tiny.parquet", 1024, 1024)); + EXPECT_FALSE(should_stage_small_http_file("https://host/large.parquet", 1025, 1024)); + EXPECT_FALSE(should_stage_small_http_file("/tmp/tiny.parquet", 512, 1024)); + EXPECT_FALSE(should_stage_small_http_file("s3://bucket/tiny.parquet", 512, 1024)); +} - ASSERT_EQ(plan.row_groups.size(), 3); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); - for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { - EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); - EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, - static_cast(row_group_idx * 2)); - ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); - EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); - } +TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { + using format::parquet::detail::MAX_NATIVE_LAZY_SKIP_ROWS; + using format::parquet::detail::bounded_native_lazy_skip_rows; + EXPECT_EQ(bounded_native_lazy_skip_rows(1), 1); + EXPECT_EQ(bounded_native_lazy_skip_rows(MAX_NATIVE_LAZY_SKIP_ROWS + 1), + MAX_NATIVE_LAZY_SKIP_ROWS); } TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { @@ -612,7 +972,8 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater .first_file_row = ConditionCacheContext::GRANULE_SIZE, .row_group_rows = ConditionCacheContext::GRANULE_SIZE, .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, - .page_skip_plans = {}}); + .page_skip_plans = {}, + .offset_indexes = {}}); format::parquet::ParquetScanScheduler scheduler; scheduler.set_plan(std::move(plan)); @@ -627,131 +988,6 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater EXPECT_EQ(ctx->base_granule, 0); } -TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { - write_page_index_pair_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest single_filter_request; - format::FileScanRequestBuilder single_filter_builder(&single_filter_request); - ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - single_filter_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - format::parquet::RowGroupScanPlan single_filter_plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - single_filter_request, scan_range, false, &single_filter_plan) - .ok()); - ASSERT_EQ(single_filter_plan.row_groups.size(), 1); - const int64_t single_filter_rows = - count_range_rows(single_filter_plan.row_groups[0].selected_ranges); - - format::FileScanRequest intersect_request; - format::FileScanRequestBuilder intersect_builder(&intersect_request); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); - format::parquet::RowGroupScanPlan intersect_plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - intersect_request, scan_range, false, &intersect_plan) - .ok()); - ASSERT_EQ(intersect_plan.row_groups.size(), 1); - ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); - const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); - EXPECT_GT(single_filter_rows, intersect_rows); - EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); - const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); - EXPECT_LT(last_range.start + last_range.length, 128); - EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, - intersect_plan.row_groups[0].selected_ranges.size()); - - auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); - EXPECT_FALSE(id_skip_plan->second.empty()); - auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); - ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); - EXPECT_FALSE(score_skip_plan->second.empty()); -} - -TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); -} - -TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); - - const bool old_enable_page_index = config::enable_parquet_page_index; - config::enable_parquet_page_index = false; - std::vector selected_ranges; - std::map page_skip_plans; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, - &page_skip_plans, &pruning_stats) - .ok()); - config::enable_parquet_page_index = old_enable_page_index; - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 128); - EXPECT_TRUE(page_skip_plans.empty()); - EXPECT_EQ(pruning_stats.page_index_read_calls, 0); - - write_int_pair_parquet_file(_file_path, 6); - auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto no_index_schema = build_file_schema(*no_index_reader); - format::FileScanRequest no_index_request; - no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - no_index_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); - selected_ranges.clear(); - page_skip_plans.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - no_index_reader.get(), no_index_schema, no_index_request, 0, 6, - &selected_ranges, &page_skip_plans, &pruning_stats) - .ok()); - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 6); - EXPECT_TRUE(page_skip_plans.empty()); -} - TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -1081,6 +1317,37 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { EXPECT_EQ(row_ids, std::vector({2, 3})); } +TEST_F(ParquetScanTest, PredicateOnlyGlobalRowIdKeepsSignedFileLocalId) { + write_int_pair_parquet_file(_file_path, 6, false); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + const auto global_rowid = format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(global_rowid).ok()); + request->predicate_only_columns.push_back(global_rowid); + const auto global_rowid_position = request->local_positions.at(global_rowid); + request->conjuncts.push_back(create_always_true_single_column_conjunct( + cast_set(global_rowid_position.value()))); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 6); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {1, 2, 3, 4, 5, 6})); +} + TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { write_int_pair_parquet_file(_file_path, 2); auto reader = create_reader(); @@ -1146,7 +1413,8 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { while (!eof) { Block block = build_file_block(schema); size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -1165,6 +1433,9 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 1); + EXPECT_GT(counter_value(profile, "PredicateCompactionBytes"), 0); + ASSERT_NE(profile.get_counter("PredicateCompactionTime"), nullptr); EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); @@ -1206,6 +1477,244 @@ TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFilte EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); } +TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedPlainComparisonUsesPhysicalFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedByteStreamSplitUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::BYTE_STREAM_SPLIT); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedDeltaBinaryPackedUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::DELTA_BINARY_PACKED); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { + write_uint32_pair_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BIGINT); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back( + create_int64_direct_greater_conjunct(0, std::numeric_limits::max())); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 2); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {20, 30})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + +TEST_F(ParquetScanTest, FixedWidthPredicateReportsUnsupportedEncodingAfterProgress) { + write_misdeclared_two_page_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("encoding"), std::string::npos) << status; +} + +TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 42)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + // A residual expression still consumes id, so raw filtering must leave its payload available. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. @@ -1236,10 +1745,12 @@ TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { } EXPECT_EQ(total_rows, 0); - EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 6); + // The first one-row probe grows after rejection and remains grown across consecutive empty + // row groups, so later two-row groups are consumed in one predicate batch each. + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); } // Scenario: row group 0 is fully filtered and leaves two pending lazy rows. Reset must discard that @@ -1281,6 +1792,85 @@ TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); } +TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { + constexpr size_t ROWS = 70000; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, ROWS - 1)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); + EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); + EXPECT_LT(counter_value(profile, "TotalBatches"), 32); +} + +TEST_F(ParquetScanTest, EmptyPrefixNeverWidensReturnedBatchPastCallerCap) { + constexpr size_t ROWS = 300; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + int32_t expected_id = 32; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + // The first 32-row probe is empty. Its feedback may accelerate later predicate work, but + // each pending-output slice must preserve the caller's memory-derived row cap. + ASSERT_LE(rows, 32); + const auto& ids = int32_data_column(*block.get_by_position(0).column); + const auto& first_lazy = int32_data_column(*block.get_by_position(1).column); + const auto& second_lazy = int32_data_column(*block.get_by_position(2).column); + ASSERT_EQ(first_lazy.size(), rows); + ASSERT_EQ(second_lazy.size(), rows); + for (size_t row = 0; row < rows; ++row, ++expected_id) { + EXPECT_EQ(ids.get_element(row), expected_id); + EXPECT_EQ(first_lazy.get_element(row), expected_id + 10); + EXPECT_EQ(second_lazy.get_element(row), expected_id + 20); + } + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS - 32); + EXPECT_EQ(expected_id, ROWS); +} + // Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume // the complete repetition/definition-level span for the first list, then materialize the remaining // two parent rows without corrupting their child boundaries. @@ -1415,5 +2005,30 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); } +TEST_F(ParquetScanTest, OpenDefersPageIndexProbeToCurrentRowGroup) { + write_multi_row_group_page_index_parquet_file(_file_path); + RuntimeProfile profile("lazy_page_index_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, -1)); + ASSERT_TRUE(reader->open(request).ok()); + + // LIMIT can stop after the first block, so open must not pay remote index I/O for later groups. + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 128); + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 1); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 9f78735bfb560a..d116f84fbaf8a0 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -16,9 +16,10 @@ // under the License. #include -#include +#include #include +#include #include #include "core/assert_cast.h" @@ -27,509 +28,582 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { -namespace { - -std::vector> build_fields( - const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - return fields; +TEST(ParquetSchemaTest, NativeMetadataAcceptsRequiredRootWithoutColumns) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(0); + root.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root}).ok()); + EXPECT_EQ(descriptor.size(), 0); + + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root}); + thrift_metadata.__set_num_rows(0); + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + // An empty physical tree remains useful for metadata-only COUNT(*); rejecting it here changes + // the compatibility contract before request planning can select that path. + EXPECT_TRUE(metadata.init_schema(false, false).ok()); + EXPECT_EQ(metadata.schema().size(), 0); } -Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - return build_parquet_column_schema(descriptor, &fields); -} +TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); -} // namespace - -TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT64, - ::parquet::ConvertedType::NONE, -1, -1, -1, 42), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->local_id, 0); - EXPECT_EQ(fields[0]->name, "required_i32"); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->leaf_column_id, 0); - EXPECT_EQ(fields[0]->nullable_definition_level, 0); - EXPECT_FALSE(fields[0]->type->is_nullable()); - - EXPECT_EQ(fields[1]->local_id, 1); - EXPECT_EQ(fields[1]->parquet_field_id, 42); - EXPECT_EQ(fields[1]->leaf_column_id, 1); - EXPECT_EQ(fields[1]->nullable_definition_level, 1); - EXPECT_TRUE(fields[1]->type->is_nullable()); -} + tparquet::SchemaElement protocol; + protocol.__set_name("protocol"); + protocol.__set_num_children(2); + protocol.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + protocol.__set_field_id(10); -TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(false, - ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64), - ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8), - ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - }); - - ASSERT_EQ(fields.size(), 3); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); - EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); - - EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); - EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); - EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); - - EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); -} + tparquet::SchemaElement min_reader; + min_reader.__set_name("minReaderVersion"); + min_reader.__set_type(tparquet::Type::INT32); + min_reader.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + min_reader.__set_field_id(11); -TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - })}); + tparquet::SchemaElement min_writer = min_reader; + min_writer.__set_name("minWriterVersion"); + min_writer.__set_field_id(12); - ASSERT_EQ(fields.size(), 1); - const auto& struct_schema = *fields[0]; - EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(struct_schema.nullable_definition_level, 1); - ASSERT_EQ(struct_schema.children.size(), 2); - EXPECT_EQ(struct_schema.children[0]->definition_level, 1); - EXPECT_EQ(struct_schema.children[1]->definition_level, 2); - EXPECT_EQ(struct_schema.max_definition_level, 2); - - const auto& struct_type = - assert_cast(*remove_nullable(struct_schema.type)); - ASSERT_EQ(struct_type.get_elements().size(), 2); - EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); - EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); -} + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, protocol, min_reader, min_writer}).ok()); + native_schema.assign_ids(); -TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { - const auto standard_list = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - const auto structural_array = ::parquet::schema::GroupNode::Make( - "ys", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "array", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({standard_list, structural_array}); - ASSERT_EQ(fields.size(), 2); - - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(xs.definition_level, 2); - EXPECT_EQ(xs.repetition_level, 1); - ASSERT_EQ(xs.children.size(), 1); - EXPECT_EQ(xs.children[0]->name, "element"); - EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_TRUE(xs.children[0]->type->is_nullable()); - const auto& xs_type = assert_cast(*remove_nullable(xs.type)); - EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); - - const auto& ys = *fields[1]; - EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(ys.children.size(), 1); - EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); -} - -TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { - const auto two_level_list = ::parquet::schema::GroupNode::Make( - "two_level", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - const auto tuple_list = ::parquet::schema::GroupNode::Make( - "tuple_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "tuple_list_tuple", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - const auto multi_field_list = ::parquet::schema::GroupNode::Make( - "records", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::LIST); - const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); - ASSERT_EQ(fields.size(), 3); - - const auto& two_level = *fields[0]; - EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(two_level.definition_level, 2); - EXPECT_EQ(two_level.repetition_level, 1); - ASSERT_EQ(two_level.children.size(), 1); - EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(two_level.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); - - const auto& tuple = *fields[1]; - ASSERT_EQ(tuple.children.size(), 1); - EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(tuple.children[0]->name, "element"); - ASSERT_EQ(tuple.children[0]->children.size(), 1); - EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); - - const auto& multi_field = *fields[2]; - ASSERT_EQ(multi_field.children.size(), 1); - EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(multi_field.children[0]->children.size(), 2); - EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); - EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); + ASSERT_EQ(fields.size(), 1); + EXPECT_EQ(fields[0]->name, "protocol"); + EXPECT_EQ(fields[0]->parquet_field_id, 10); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(fields[0]->children[0]->name, "minReaderVersion"); + EXPECT_EQ(fields[0]->children[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->children[1]->name, "minWriterVersion"); + EXPECT_EQ(fields[0]->children[1]->leaf_column_id, 1); + + std::shared_ptr mapping; + ASSERT_TRUE(build_native_schema_node(fields[0]->type, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("minReaderVersion")); + EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); + ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } - -TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { - const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( - "outer", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({list_with_repeated_child}); +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement adjusted_time; + adjusted_time.__set_name("time_ms"); + adjusted_time.__set_type(tparquet::Type::INT32); + adjusted_time.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + adjusted_time.__set_logicalType(tparquet::LogicalType()); + adjusted_time.logicalType.__set_TIME(tparquet::TimeType()); + adjusted_time.logicalType.TIME.__set_isAdjustedToUTC(true); + adjusted_time.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + adjusted_time.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, adjusted_time}).ok()); + native_schema.assign_ids(); + std::vector> fields; + const auto status = build_parquet_column_schema(native_schema, &fields); + ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(fields.size(), 1); - const auto& outer = *fields[0]; - EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(outer.children.size(), 1); - const auto& element = *outer.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.children[0]->name, "items"); - ASSERT_EQ(element.children[0]->children.size(), 1); - EXPECT_EQ(element.children[0]->children[0]->name, "element"); + // Preserve the unsupported marker in metadata; request-level validation decides whether the + // leaf is a real projection or an ignorable COUNT(*) placeholder. + EXPECT_EQ(fields[0]->type_descriptor.unsupported_reason, + "Parquet TIME with isAdjustedToUTC=true is not supported"); } -TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { - const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); - - const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({nested_list_wrapper}); +TEST(ParquetSchemaTest, NativeOversizedByteArrayDecimalIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement decimal; + decimal.__set_name("decimal_77"); + decimal.__set_type(tparquet::Type::BYTE_ARRAY); + decimal.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + decimal.__set_logicalType(tparquet::LogicalType()); + decimal.logicalType.__set_DECIMAL(tparquet::DecimalType()); + decimal.logicalType.DECIMAL.__set_precision(77); + decimal.logicalType.DECIMAL.__set_scale(2); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, decimal}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(xs.children.size(), 1); - const auto& element = *xs.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.name, "element"); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); + EXPECT_TRUE(fields[0]->type_descriptor.is_decimal); + EXPECT_FALSE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find("precision 77"), + std::string::npos); } -TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); - +TEST(ParquetSchemaTest, NativeUnknownLogicalTypeRetainsPhysicalFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement interval; + interval.__set_name("duration"); + interval.__set_type(tparquet::Type::FIXED_LEN_BYTE_ARRAY); + interval.__set_type_length(12); + interval.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + interval.__set_logicalType(tparquet::LogicalType()); + interval.__set_converted_type(tparquet::ConvertedType::INTERVAL); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, interval}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); - EXPECT_EQ(map_schema.definition_level, 2); - EXPECT_EQ(map_schema.repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->name, "key"); - EXPECT_EQ(map_schema.children[1]->name, "value"); - EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); + EXPECT_TRUE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_EQ(fields[0]->type_descriptor.doris_type->get_primitive_type(), TYPE_STRING); } -TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::REQUIRED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); +TEST(ParquetSchemaTest, NativeGroupPrimitiveLogicalTypesAreRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement child; + child.__set_name("value"); + child.__set_type(tparquet::Type::BYTE_ARRAY); + child.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + std::vector> logical_types; + tparquet::LogicalType string_type; + string_type.__set_STRING(tparquet::StringType()); + logical_types.emplace_back("STRING", string_type); + tparquet::LogicalType enum_type; + enum_type.__set_ENUM(tparquet::EnumType()); + logical_types.emplace_back("ENUM", enum_type); + tparquet::LogicalType decimal_type; + decimal_type.__set_DECIMAL(tparquet::DecimalType()); + logical_types.emplace_back("DECIMAL", decimal_type); + tparquet::LogicalType date_type; + date_type.__set_DATE(tparquet::DateType()); + logical_types.emplace_back("DATE", date_type); + tparquet::LogicalType time_type; + time_type.__set_TIME(tparquet::TimeType()); + logical_types.emplace_back("TIME", time_type); + tparquet::LogicalType timestamp_type; + timestamp_type.__set_TIMESTAMP(tparquet::TimestampType()); + logical_types.emplace_back("TIMESTAMP", timestamp_type); + tparquet::LogicalType integer_type; + integer_type.__set_INTEGER(tparquet::IntType()); + logical_types.emplace_back("INTEGER", integer_type); + tparquet::LogicalType uuid_type; + uuid_type.__set_UUID(tparquet::UUIDType()); + logical_types.emplace_back("UUID", uuid_type); + tparquet::LogicalType float16_type; + float16_type.__set_FLOAT16(tparquet::Float16Type()); + logical_types.emplace_back("FLOAT16", float16_type); + + for (const auto& [name, logical_type] : logical_types) { + SCOPED_TRACE(name); + tparquet::SchemaElement group; + group.__set_name("bad_" + name + "_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_logicalType(logical_type); + NativeFieldDescriptor native_schema; + const auto status = native_schema.parse_from_thrift({root, group, child}); + EXPECT_FALSE(status.ok()); + if (name == "ENUM") { + EXPECT_NE(status.to_string().find("Logical type Enum cannot be applied to group node"), + std::string::npos); + } + } + + for (const auto converted_type : + {tparquet::ConvertedType::UTF8, tparquet::ConvertedType::ENUM, + tparquet::ConvertedType::DECIMAL, tparquet::ConvertedType::DATE, + tparquet::ConvertedType::TIME_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS, + tparquet::ConvertedType::INT_32, tparquet::ConvertedType::JSON, + tparquet::ConvertedType::BSON}) { + SCOPED_TRACE(tparquet::to_string(converted_type)); + tparquet::SchemaElement group; + group.__set_name("bad_converted_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_converted_type(converted_type); + NativeFieldDescriptor native_schema; + EXPECT_FALSE(native_schema.parse_from_thrift({root, group, child}).ok()); + } +} - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_FALSE(map_schema.type->is_nullable()); - EXPECT_EQ(map_schema.definition_level, 1); - EXPECT_EQ(map_schema.repetition_level, 1); - EXPECT_EQ(map_schema.repeated_repetition_level, 1); - EXPECT_EQ(map_schema.max_definition_level, 2); - EXPECT_EQ(map_schema.max_repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->definition_level, 1); - EXPECT_EQ(map_schema.children[0]->repetition_level, 1); - EXPECT_EQ(map_schema.children[1]->definition_level, 2); - EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeFlatLeafValueCountMustMatchRowCount) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::Statistics statistics; + statistics.__set_null_count(2); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(2); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(1); + thrift_metadata.__set_row_groups({row_group}); + + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + const auto status = metadata.init_schema(false, false); + EXPECT_TRUE(status.is()) << status; } -TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32), - ::parquet::schema::GroupNode::Make( - "links", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[0]->children.size(), 1); - EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->children[0]->name, "element"); - - EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[1]->children.size(), 1); - EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(fields[1]->children[0]->name, "element"); +TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(8); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return leaf; + }; + auto logical_enum = binary_leaf("logical_enum"); + logical_enum.__set_logicalType(tparquet::LogicalType()); + logical_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto logical_bson = binary_leaf("logical_bson"); + logical_bson.__set_logicalType(tparquet::LogicalType()); + logical_bson.logicalType.__set_BSON(tparquet::BsonType()); + auto converted_enum = binary_leaf("converted_enum"); + converted_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto converted_bson = binary_leaf("converted_bson"); + converted_bson.__set_converted_type(tparquet::ConvertedType::BSON); + + auto logical_time = [](const std::string& name, bool millis) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(millis ? tparquet::Type::INT32 : tparquet::Type::INT64); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + leaf.__set_logicalType(tparquet::LogicalType()); + leaf.logicalType.__set_TIME(tparquet::TimeType()); + leaf.logicalType.TIME.__set_isAdjustedToUTC(false); + leaf.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + if (millis) { + leaf.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + } else { + leaf.logicalType.TIME.unit.__set_MICROS(tparquet::MicroSeconds()); + } + return leaf; + }; + auto logical_millis = logical_time("logical_millis", true); + auto logical_micros = logical_time("logical_micros", false); + auto converted_millis = logical_time("converted_millis", true); + converted_millis.__isset.logicalType = false; + converted_millis.__set_converted_type(tparquet::ConvertedType::TIME_MILLIS); + auto converted_micros = logical_time("converted_micros", false); + converted_micros.__isset.logicalType = false; + converted_micros.__set_converted_type(tparquet::ConvertedType::TIME_MICROS); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift({root, logical_enum, logical_bson, converted_enum, + converted_bson, logical_millis, logical_micros, + converted_millis, converted_micros}) + .ok()); + for (size_t field = 0; field < 4; ++field) { + EXPECT_EQ(remove_nullable(descriptor.get_column(field)->data_type)->get_primitive_type(), + TYPE_STRING); + } + EXPECT_EQ(remove_nullable(descriptor.get_column(4)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(5)->data_type)->get_scale(), 6); + EXPECT_EQ(remove_nullable(descriptor.get_column(6)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); } -TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "inner", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); +TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { + auto valid_root = []() { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + return root; + }; + auto valid_leaf = []() { + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + return leaf; + }; + + NativeFieldDescriptor descriptor; + auto primitive_root = valid_root(); + primitive_root.__set_type(tparquet::Type::INT32); + EXPECT_FALSE(descriptor.parse_from_thrift({primitive_root, valid_leaf()}).ok()); + + auto repeated_root = valid_root(); + repeated_root.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + EXPECT_FALSE(descriptor.parse_from_thrift({repeated_root, valid_leaf()}).ok()); + + auto missing_repetition = valid_leaf(); + missing_repetition.__isset.repetition_type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_repetition}).ok()); + + auto dual_kind = valid_leaf(); + dual_kind.__set_num_children(1); + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + + auto legacy_zero_children = valid_leaf(); + legacy_zero_children.__set_num_children(0); + EXPECT_TRUE(descriptor.parse_from_thrift({valid_root(), legacy_zero_children}).ok()); + + auto missing_kind = valid_leaf(); + missing_kind.__isset.type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); +} - ASSERT_EQ(fields.size(), 1); - const auto& s = *fields[0]; - EXPECT_EQ(s.definition_level, 1); - EXPECT_EQ(s.nullable_definition_level, 1); - ASSERT_EQ(s.children.size(), 1); - const auto& inner = *s.children[0]; - EXPECT_EQ(inner.definition_level, 2); - EXPECT_EQ(inner.nullable_definition_level, 2); - ASSERT_EQ(inner.children.size(), 1); - const auto& items = *inner.children[0]; - EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(items.definition_level, 3); - EXPECT_EQ(items.repetition_level, 1); - EXPECT_EQ(items.repeated_ancestor_definition_level, 3); - EXPECT_EQ(items.repeated_repetition_level, 1); - EXPECT_EQ(items.max_definition_level, 3); - EXPECT_EQ(items.max_repetition_level, 1); - ASSERT_EQ(items.children.size(), 1); - EXPECT_EQ(items.children[0]->definition_level, 3); - EXPECT_EQ(items.children[0]->repetition_level, 1); +TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(std::numeric_limits::max()); + NativeFieldDescriptor descriptor; + EXPECT_FALSE(descriptor.parse_from_thrift({root}).ok()); + + root.__set_num_children(1); + tparquet::SchemaElement nested; + nested.__set_name("nested"); + nested.__set_num_children(std::numeric_limits::max()); + nested.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + EXPECT_FALSE(descriptor.parse_from_thrift({root, nested}).ok()); } -TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { - auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, - ::parquet::schema::NodeVector {}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(empty_root); +std::vector nested_native_schema(size_t depth) { + std::vector schema; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + schema.push_back(root); + for (size_t level = 0; level < depth; ++level) { + tparquet::SchemaElement group; + group.__set_name("g" + std::to_string(level)); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(group); + } + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(leaf); + return schema; +} - EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); +TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { + NativeFieldDescriptor accepted; + EXPECT_TRUE(accepted.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH)).ok()); + NativeFieldDescriptor rejected; + EXPECT_FALSE( + rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); +} - std::vector> fields; - ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - EXPECT_TRUE(fields.empty()); +TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto list = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema.__set_converted_type(tparquet::ConvertedType::LIST); + return schema; + }; + auto wrapper = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + return schema; + }; + auto item = []() { + tparquet::SchemaElement schema; + schema.__set_name("item"); + schema.__set_type(tparquet::Type::INT32); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return schema; + }; + + NativeFieldDescriptor mismatched; + ASSERT_TRUE(mismatched.parse_from_thrift({root(), list("xs"), wrapper("other_tuple"), item()}) + .ok()); + const auto* mismatched_element = &mismatched.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); + + NativeFieldDescriptor matching; + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}).ok()); + const auto* matching_element = &matching.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(matching_element->children.size(), 1); + EXPECT_EQ(matching_element->children[0].name, "item"); } -TEST(ParquetSchemaTest, RejectInvalidListMapAndPreserveUnsupportedTime) { - const auto bad_list = ::parquet::schema::GroupNode::Make( - "bad_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({bad_list}).ok()); - - const auto bad_map = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({bad_map}).ok()); - - const auto converted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS); - const auto fields = build_fields({converted_time}); - ASSERT_EQ(fields.size(), 1); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find( - "Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); +TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto group = [](const std::string& name, tparquet::FieldRepetitionType::type repetition) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(repetition); + return schema; + }; + auto outer_list = group("outer", tparquet::FieldRepetitionType::OPTIONAL); + outer_list.__set_converted_type(tparquet::ConvertedType::LIST); + + auto repeated_wrapper = group("list", tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement repeated_items; + repeated_items.__set_name("items"); + repeated_items.__set_type(tparquet::Type::INT32); + repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + NativeFieldDescriptor repeated; + ASSERT_TRUE(repeated.parse_from_thrift({root(), outer_list, repeated_wrapper, repeated_items}) + .ok()); + const auto* repeated_element = &repeated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(repeated_element->children.size(), 1); + EXPECT_EQ(repeated_element->children[0].name, "items"); + EXPECT_EQ(remove_nullable(repeated_element->children[0].data_type)->get_primitive_type(), + TYPE_ARRAY); + + auto annotated_wrapper = repeated_wrapper; + annotated_wrapper.__set_converted_type(tparquet::ConvertedType::LIST); + auto nested_wrapper = repeated_wrapper; + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT32); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + NativeFieldDescriptor annotated; + ASSERT_TRUE(annotated + .parse_from_thrift( + {root(), outer_list, annotated_wrapper, nested_wrapper, value}) + .ok()); + const auto* annotated_element = &annotated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(annotated_element->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(annotated_element->children.size(), 1); + EXPECT_EQ(remove_nullable(annotated_element->children[0].data_type)->get_primitive_type(), + TYPE_INT); } -TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { - const auto zero_child_list = ::parquet::schema::GroupNode::Make( - "zero_child_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, - ::parquet::schema::NodeVector {})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({zero_child_list}).ok()); - - const auto repeated_list = ::parquet::schema::GroupNode::Make( - "repeated_list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({repeated_list}).ok()); - - const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::GroupNode::Make( - "entry1", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - ::parquet::schema::GroupNode::Make( - "entry2", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_with_two_fields}).ok()); - - const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); - - const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); - - const auto repeated_map = ::parquet::schema::GroupNode::Make( - "repeated_map", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({repeated_map}).ok()); +TEST(ParquetSchemaTest, NativeSchemaRecognizesLogicalTypeOnlyListAndMap) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + + tparquet::SchemaElement list; + list.__set_name("items"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_logicalType(tparquet::LogicalType()); + list.logicalType.__set_LIST(tparquet::ListType()); + tparquet::SchemaElement list_wrapper; + list_wrapper.__set_name("list"); + list_wrapper.__set_num_children(1); + list_wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_type(tparquet::Type::INT32); + element.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::SchemaElement map; + map.__set_name("attributes"); + map.__set_num_children(1); + map.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + map.__set_logicalType(tparquet::LogicalType()); + map.logicalType.__set_MAP(tparquet::MapType()); + tparquet::SchemaElement key_value; + key_value.__set_name("key_value"); + key_value.__set_num_children(2); + key_value.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::BYTE_ARRAY); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT64); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift( + {root, list, list_wrapper, element, map, key_value, key, value}) + .ok()); + ASSERT_EQ(descriptor.size(), 2); + EXPECT_EQ(remove_nullable(descriptor.get_column(0)->data_type)->get_primitive_type(), + TYPE_ARRAY); + EXPECT_EQ(remove_nullable(descriptor.get_column(1)->data_type)->get_primitive_type(), TYPE_MAP); } -TEST(ParquetSchemaTest, LogicalUtcTimeIsPreservedForProjection) { - const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32); - const auto supported_value = ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64); - const auto row = ::parquet::schema::GroupNode::Make("row", ::parquet::Repetition::OPTIONAL, - {adjusted_time, supported_value}); - const auto fields = build_fields({row}); - ASSERT_EQ(fields.size(), 1); - ASSERT_EQ(fields[0]->children.size(), 2); - EXPECT_EQ(remove_nullable(fields[0]->children[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_FALSE(fields[0]->children[0]->type_descriptor.unsupported_reason.empty()); - EXPECT_EQ(remove_nullable(fields[0]->children[1]->type)->get_primitive_type(), TYPE_BIGINT); +TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { + auto make_metadata = []() { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + tparquet::SchemaElement first; + first.__set_name("a"); + first.__set_type(tparquet::Type::INT32); + first.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement second = first; + second.__set_name("b"); + metadata.__set_schema({root, first, second}); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + return metadata; + }; + + { + NativeParquetMetadata metadata(make_metadata(), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } + { + auto thrift = make_metadata(); + thrift.row_groups[0].columns.resize(2); + tparquet::ColumnMetaData first_meta; + first_meta.__set_type(tparquet::Type::INT32); + thrift.row_groups[0].columns[0].__set_meta_data(first_meta); + NativeParquetMetadata metadata(std::move(thrift), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } } } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp deleted file mode 100644 index c35138e3263723..00000000000000 --- a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/types.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -class ParquetSerdeReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "serde.parquet").string(); - write_parquet_file(); - open_file(_file_path); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(0x0000).ok()); - EXPECT_TRUE(builder.Append(0x8000).ok()); - EXPECT_TRUE(builder.Append(0x3E00).ok()); - EXPECT_TRUE(builder.Append(0x7E00).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_binary_array(const std::vector& values) { - arrow::BinaryBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), - static_cast(value.size())) - .ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_fixed_binary_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_timestamp_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_decimal_array(const std::shared_ptr& type, - const std::vector& values) { - arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); - } - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, - std::shared_ptr array) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - } - - void write_table(const std::string& file_path, const std::shared_ptr& table, - std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - ::parquet::WriterProperties::Builder writer_builder; - writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - writer_builder.compression(::parquet::Compression::UNCOMPRESSED); - if (arrow_properties == nullptr) { - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_properties = arrow_builder.build(); - } - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, - writer_builder.build(), std::move(arrow_properties))); - } - - void write_parquet_file() { - add_field(arrow::field("bool_col", arrow::boolean(), false), - build_required_array( - {true, false, true, false, true})); - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50})); - add_field(arrow::field("int64_col", arrow::int64(), false), - build_required_array( - {10000000000L, -9L, 42L, 77L, 123L})); - add_field(arrow::field("uint32_col", arrow::uint32(), false), - build_required_array( - {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); - add_field(arrow::field("uint64_col", arrow::uint64(), false), - build_required_array( - {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); - add_field(arrow::field("float_col", arrow::float32(), false), - build_required_array( - {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); - add_field(arrow::field("double_col", arrow::float64(), false), - build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); - add_field(arrow::field("nullable_float16_col", arrow::float16(), true), - build_nullable_float16_array()); - add_field(arrow::field("binary_col", arrow::binary(), false), - build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); - add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), - build_fixed_binary_array(arrow::fixed_size_binary(4), - {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); - add_field(arrow::field("date_col", arrow::date32(), false), - build_required_array({0, 1, 18628, 18629, 18630})); - add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), - {0, 1234, 1609459200000, 1609459201000, -1})); - add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("timestamp_micros_utc_col", - arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), - build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); - add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), - build_decimal_array(arrow::decimal128(18, 6), - {1234567, -670000, 0, 9870000, 1000000})); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array()); - - write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); - } - - void open_file(const std::string& file_path) { - _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); - ASSERT_NE(_file_reader, nullptr); - ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet serde test field " << name; - return _fields.size(); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - template - void read_and_validate(const std::string& name, Validator validator) const { - const auto field_idx = find_field_idx(name); - ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - validator(*_fields[field_idx], *column); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; -}; - -TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { - read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 0); - EXPECT_EQ(values.get_element(4), 1); - }); - read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10000000000L); - EXPECT_EQ(values.get_element(1), -9L); - }); - read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), 2147483648L); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), static_cast(1) << 63); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); - const auto& values = assert_cast(column); - EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); - EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); - }); - read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); - const auto& values = assert_cast(column); - EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); - EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); - }); - read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 2); - EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(schema.type_descriptor.is_string_like); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); - const auto& nullable_column = assert_cast(column); - const auto& values = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); - EXPECT_FALSE(std::signbit(values.get_element(1))); - EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); - EXPECT_TRUE(std::signbit(values.get_element(2))); - EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); - EXPECT_TRUE(std::isnan(values.get_element(4))); - }); - read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); - EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); - }); - read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 4); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); - EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); - }); - read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); - EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); - EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); - }); - read_and_validate( - "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); - }); - read_and_validate( - "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal32(12345)); - EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); - }); - read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal64(1234567)); - EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); - }); - read_and_validate( - "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); -} - -TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { - const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); - auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); - auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1}); - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_builder.enable_force_write_int96_timestamps(); - _fields.clear(); - _file_reader.reset(); - _row_group.reset(); - write_table(file_path, table, arrow_builder.build()); - open_file(file_path); - - ASSERT_EQ(_fields.size(), 1); - EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); - EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); - ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - - auto reader = create_reader(0); - ASSERT_NE(reader, nullptr); - auto column = _fields[0]->type->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); - ASSERT_EQ(rows_read, ROW_COUNT); - EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index dd2138279b11d1..e69f1311e4ab2b 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -17,29 +17,26 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include #include -#include -#include -#include -#include -#include #include #include #include +#include #include #include #include #include +#include #include #include +#include "core/data_type/data_type_date.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_time.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -47,293 +44,157 @@ #include "exprs/vslot_ref.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "storage/index/bloom_filter/block_split_bloom_filter.h" -#include "storage/index/zone_map/zonemap_eval_context.h" -#include "storage/index/zone_map/zonemap_filter_result.h" - +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "io/fs/file_reader.h" +#include "util/thrift_util.h" namespace doris { namespace { -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -std::shared_ptr int32_array(const std::vector>& values) { - arrow::Int32Builder builder; - for (const auto& value : values) { - if (value.has_value()) { - EXPECT_TRUE(builder.Append(*value).ok()); - } else { - EXPECT_TRUE(builder.AppendNull().ok()); - } - } - return finish_array(&builder); -} - -std::shared_ptr uint32_array(const std::vector& values) { - arrow::UInt32Builder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr float_array(const std::vector& values) { - arrow::FloatBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr double_array(const std::vector& values) { - arrow::DoubleBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -template -std::string encoded_value(const NativeType& value) { - return {reinterpret_cast(&value), sizeof(value)}; -} - -std::shared_ptr string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr timestamp_array(const std::vector& values) { - arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::unique_ptr<::parquet::ParquetFileReader> make_reader( - const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, - bool enable_statistics) { - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.compression(::parquet::Compression::UNCOMPRESSED); - if (enable_dictionary) { - builder.enable_dictionary(); - } else { - builder.disable_dictionary(); - } - if (!enable_statistics) { - builder.disable_statistics(); - } - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - row_group_size, builder.build()) - .ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - return ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); -} - -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) - .ok()); - return file_schema; -} - -template -class TestColumnIndex final : public ::parquet::TypedColumnIndex { +class StatisticsMemoryFileReader final : public io::FileReader { public: - using NativeType = typename ParquetDType::c_type; - - TestColumnIndex(NativeType min_value, NativeType max_value) - : TestColumnIndex(std::vector {min_value}, - std::vector {max_value}) {} - - TestColumnIndex(std::vector min_values, std::vector max_values) - : _null_pages(min_values.size(), false), - _null_counts(min_values.size(), 0), - _min_values(std::move(min_values)), - _max_values(std::move(max_values)) { - EXPECT_EQ(_min_values.size(), _max_values.size()); - for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { - _non_null_page_indices.push_back(static_cast(page_idx)); - } - } + explicit StatisticsMemoryFileReader(std::vector bytes) + : _bytes(std::move(bytes)), _path("native-bloom-filter.parquet") {} - const std::vector& null_pages() const override { return _null_pages; } - const std::vector& encoded_min_values() const override { return _encoded_values; } - const std::vector& encoded_max_values() const override { return _encoded_values; } - ::parquet::BoundaryOrder::type boundary_order() const override { - return ::parquet::BoundaryOrder::Unordered; + Status close() override { + _closed = true; + return Status::OK(); } - bool has_null_counts() const override { return true; } - const std::vector& null_counts() const override { return _null_counts; } - const std::vector& non_null_page_indices() const override { - return _non_null_page_indices; + const io::Path& path() const override { return _path; } + size_t size() const override { return _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _bytes.size() || result.size > _bytes.size() - offset) { + return Status::IOError("native Bloom test read exceeds memory file"); + } + memcpy(result.data, _bytes.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); } - const std::vector& min_values() const override { return _min_values; } - const std::vector& max_values() const override { return _max_values; } private: - const std::vector _null_pages; - const std::vector _encoded_values; - const std::vector _null_counts; - std::vector _non_null_page_indices; - const std::vector _min_values; - const std::vector _max_values; + std::vector _bytes; + io::Path _path; + bool _closed = false; }; - -class Int32ZoneMapExpr final : public VExpr { +class BloomInExpr final : public VExpr { public: - enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; - - Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) + BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) : VExpr(std::make_shared(), false), - _column_id(column_id), - _op(op), - _value(value) {} + _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), + _values(std::move(values)) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); + return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); } - bool can_evaluate_zonemap_filter() const override { return true; } + bool can_evaluate_bloom_filter() const override { return true; } - void collect_slot_column_ids(std::set& column_ids) const override { - column_ids.insert(_column_id); + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); } - ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { - auto zone_map = ctx.zone_map(_column_id); - if (zone_map == nullptr) { - return unsupported_zonemap_filter(ctx); - } - if (_op == Op::IS_NULL) { - return zone_map->has_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (_op == Op::IS_NOT_NULL) { - return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (!zone_map->has_not_null) { - return ZoneMapFilterResult::kNoMatch; - } - const auto literal = Field::create_field(_value); - if (_op == Op::GE) { - return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); } private: - int _column_id; - Op _op; - int32_t _value; - const std::string _expr_name = "Int32ZoneMapExpr"; + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "BloomInExpr"; }; -class StringDictionaryInExpr final : public VExpr { +class DictionaryStringInExpr final : public VExpr { public: - StringDictionaryInExpr(int column_id, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), - "c0")) { - _values.reserve(values.size()); - for (auto& value : values) { - _values.emplace_back(Field::create_field(std::move(value))); - } - } + DictionaryStringInExpr() : VExpr(std::make_shared(), false) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError( - "StringDictionaryInExpr is only used by parquet statistics tests"); + return Status::InternalError("DictionaryStringInExpr is metadata-only"); } bool can_evaluate_dictionary_filter() const override { return true; } - ZoneMapFilterResult evaluate_dictionary_filter( - const DictionaryEvalContext& ctx) const override { - return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext&) const override { + return ZoneMapFilterResult::kNoMatch; } - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); - } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "StringDictionaryInExpr"; + const std::string _expr_name = "DictionaryStringInExpr"; }; -class BloomInExpr final : public VExpr { +class MetadataInt32GreaterThanExpr final : public VExpr { public: - BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), - _values(std::move(values)) {} + explicit MetadataInt32GreaterThanExpr(int32_t value) + : VExpr(std::make_shared(), false), _value(value) {} const std::string& expr_name() const override { return _expr_name; } - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); - } - - bool can_evaluate_bloom_filter() const override { return true; } - - ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { - return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); + return Status::InternalError("MetadataInt32GreaterThanExpr is metadata-only"); } - - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + return zone_map->max_value <= Field::create_field(_value) + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "BloomInExpr"; + int32_t _value; + const std::string _expr_name = "MetadataInt32GreaterThanExpr"; }; -format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); - return request; -} +class MetadataBoundsProbeExpr final : public VExpr { +public: + explicit MetadataBoundsProbeExpr(bool require_false_boolean = false) + : VExpr(std::make_shared(), false), + _require_false_boolean(require_false_boolean) {} -format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared( - std::make_shared(0, std::move(values)))); - return request; -} + const std::string& expr_name() const override { return _expr_name; } + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("MetadataBoundsProbeExpr is metadata-only"); + } + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr || !zone_map->has_not_null || zone_map->min_value.is_null() || + zone_map->max_value.is_null()) { + return ZoneMapFilterResult::kMayMatch; + } + if (_require_false_boolean && + zone_map->min_value == Field::create_field(false) && + zone_map->max_value == Field::create_field(false)) { + return ZoneMapFilterResult::kMayMatch; + } + return ZoneMapFilterResult::kNoMatch; + } +private: + bool _require_false_boolean; + const std::string _expr_name = "MetadataBoundsProbeExpr"; +}; VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -346,690 +207,514 @@ format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); return request; } - -void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, - PrimitiveType type) { - DORIS_CHECK(bloom_filter != nullptr); - switch (type) { - case TYPE_BOOLEAN: { - const bool typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_INT: { - const int32_t typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_STRING: { - const auto& typed_value = value.get(); - bloom_filter->add_bytes(typed_value.data(), typed_value.size()); - break; - } - default: - DORIS_CHECK(false); - } -} - -std::unique_ptr bloom_filter_for_fields( - const std::vector& values, PrimitiveType type) { - auto bloom_filter = std::make_unique(); - EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); - for (const auto& value : values) { - add_bloom_field(bloom_filter.get(), value, type); - } - return bloom_filter; -} - -BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, - const segment_v2::BloomFilter* bloom_filter) { - BloomFilterEvalContext ctx; - ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, - .bloom_filter = bloom_filter}); - return ctx; -} - -std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { - auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); - bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); - return bloom_filter; -} - format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { format::parquet::ParquetColumnSchema column_schema; column_schema.type = std::make_shared(); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; column_schema.type_descriptor.integer_bit_width = 32; column_schema.type_descriptor.is_unsigned_integer = true; return column_schema; } -format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { +TEST(NativeParquetStatisticsTest, InvalidNullableDateBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared()); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = fixed_length; - column_schema.type_descriptor.is_string_like = true; - return column_schema; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + + const int32_t invalid_date = std::numeric_limits::min(); + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { +TEST(NativeParquetStatisticsTest, InvalidNullableDecimalBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared(2, 0)); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = 2; - column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; - return column_schema; -} - -TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { - auto table = arrow::Table::Make( - arrow::schema({ - arrow::field("i", arrow::int32(), true), - arrow::field("u", arrow::uint32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - }), - {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), - string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto row_group = reader->metadata()->RowGroup(0); - - const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], row_group->ColumnChunk(0)->statistics()); - EXPECT_TRUE(int_stats.has_min_max); - EXPECT_TRUE(int_stats.has_null_count); - EXPECT_TRUE(int_stats.has_null); - EXPECT_TRUE(int_stats.has_not_null); - EXPECT_EQ(int_stats.min_value.get(), 1); - EXPECT_EQ(int_stats.max_value.get(), 5); - - const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[1], row_group->ColumnChunk(1)->statistics()); - EXPECT_TRUE(uint_stats.has_min_max); - EXPECT_EQ(uint_stats.min_value.get(), 7); - EXPECT_EQ(uint_stats.max_value.get(), 11); - - const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], row_group->ColumnChunk(2)->statistics()); - EXPECT_TRUE(string_stats.has_min_max); - EXPECT_EQ(string_stats.min_value.get(), "alpha"); - EXPECT_EQ(string_stats.max_value.get(), "omega"); - - auto utc = cctz::utc_time_zone(); - const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); - EXPECT_TRUE(timestamp_stats.has_min_max); - EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); - EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); - EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); -} + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.is_decimal = true; + column_schema.type_descriptor.decimal_precision = 2; + column_schema.type_descriptor.decimal_scale = 0; -TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. - // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover - // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, - 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(local_stats.has_not_null); - EXPECT_FALSE(local_stats.has_min_max); - - auto utc = cctz::utc_time_zone(); - const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &utc); - EXPECT_TRUE(utc_stats.has_min_max); - EXPECT_LT(utc_stats.min_value, utc_stats.max_value); -} + const int32_t invalid_decimal = 1000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); -TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp - // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. - schema[0]->type = std::make_shared(6); - schema[0]->type_descriptor.doris_type = schema[0]->type; - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto timestamp_tz_stats = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(timestamp_tz_stats.has_min_max); - EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { - auto no_stats_table = arrow::Table::Make( - arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); - auto no_stats_reader = make_reader(no_stats_table, 3, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *no_stats_schema[0], - no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_FALSE(no_stats.has_min_max); - - auto all_null_table = - arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt})}); - auto all_null_reader = make_reader(all_null_table, 2, false, true); - auto all_null_schema = build_file_schema(*all_null_reader); - auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *all_null_schema[0], - all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_TRUE(all_null_stats.has_null_count); - EXPECT_TRUE(all_null_stats.has_null); - EXPECT_FALSE(all_null_stats.has_not_null); - EXPECT_FALSE(all_null_stats.has_min_max); -} +TEST(NativeParquetStatisticsTest, InvalidTimeBoundsDisableFooterMinMax) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(3); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; -TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1, std::nullopt, 3})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( - reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), - file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); - - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics_without_null_count); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_TRUE(statistics.has_null); - EXPECT_TRUE(statistics.has_not_null); - EXPECT_TRUE(statistics.has_min_max); - EXPECT_EQ(statistics.min_value.get(), 1); - EXPECT_EQ(statistics.max_value.get(), 3); -} + const int32_t invalid_time = 90000000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const float float_nan = std::numeric_limits::quiet_NaN(); - const float float_max = 2.0F; - auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( - schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, - true, true, false); - const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], float_stats); - EXPECT_FALSE(converted_float.has_min_max); - EXPECT_TRUE(converted_float.has_not_null); - - const double double_nan = std::numeric_limits::quiet_NaN(); - const double double_min = 1.0; - auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, - true, true, false); - const auto converted_double = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - double_stats); - EXPECT_FALSE(converted_double.has_min_max); - EXPECT_TRUE(converted_double.has_not_null); - - const double double_max = 2.0; - auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, - true, true, false); - const auto converted_finite = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - finite_stats); - EXPECT_TRUE(converted_finite.has_min_max); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - auto float_index = std::make_shared>( - 1.0F, std::numeric_limits::quiet_NaN()); - format::parquet::ParquetColumnStatistics float_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - float_index, *schema[0], 0, &float_page_stats)); - EXPECT_FALSE(float_page_stats.has_min_max); - EXPECT_TRUE(float_page_stats.has_not_null); - - auto double_index = std::make_shared>( - std::numeric_limits::quiet_NaN(), 2.0); - format::parquet::ParquetColumnStatistics double_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - double_index, *schema[1], 0, &double_page_stats)); - EXPECT_FALSE(double_page_stats.has_min_max); - EXPECT_TRUE(double_page_stats.has_not_null); - - auto finite_index = std::make_shared>(1.0, 2.0); - format::parquet::ParquetColumnStatistics finite_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - finite_index, *schema[1], 0, &finite_page_stats)); - EXPECT_TRUE(finite_page_stats.has_min_max); - - auto mixed_index = std::make_shared>( - std::vector {std::numeric_limits::quiet_NaN(), 1.0}, - std::vector {std::numeric_limits::quiet_NaN(), 2.0}); - format::parquet::ParquetColumnStatistics nan_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 0, &nan_page_stats)); - EXPECT_FALSE(nan_page_stats.has_min_max); - - format::parquet::ParquetColumnStatistics following_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 1, &following_page_stats)); - EXPECT_TRUE(following_page_stats.has_min_max); - EXPECT_EQ(following_page_stats.min_value.get(), 1.0); - EXPECT_EQ(following_page_stats.max_value.get(), 2.0); -} +TEST(NativeParquetStatisticsTest, BooleanFooterBoundsDecodeOnlyTheValueBit) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::BOOLEAN; + + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value(std::string(1, '\x02')); + statistics.__set_max_value(std::string(1, '\x02')); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + ASSERT_TRUE(result.has_min_max); + EXPECT_EQ(result.min_value, Field::create_field(false)); + EXPECT_EQ(result.max_value, Field::create_field(false)); +} + +TEST(NativeParquetStatisticsTest, InvalidTimeAndPaddedBooleanPageBoundsCannotPrune) { + auto run_page_index = [](std::unique_ptr column_schema, + std::string min_value, std::string max_value, + bool require_false_boolean) { + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared( + std::make_shared(require_false_boolean))}; + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({std::move(min_value)}); + page_index.column_index.__set_max_values({std::move(max_value)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + EXPECT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, + &skip_plans, nullptr) + .ok()); + return selected_ranges; + }; + + auto time_schema = std::make_unique(); + time_schema->type = std::make_shared(3); + time_schema->type_descriptor.doris_type = time_schema->type; + time_schema->type_descriptor.physical_type = tparquet::Type::INT32; + time_schema->type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; + const int32_t invalid_time = 90000000; + const std::string invalid_time_bytes(reinterpret_cast(&invalid_time), + sizeof(invalid_time)); + const auto time_ranges = + run_page_index(std::move(time_schema), invalid_time_bytes, invalid_time_bytes, false); + ASSERT_EQ(time_ranges.size(), 1); + EXPECT_EQ(time_ranges[0].start, 0); + EXPECT_EQ(time_ranges[0].length, 1); + + auto bool_schema = std::make_unique(); + bool_schema->type = std::make_shared(); + bool_schema->type_descriptor.doris_type = bool_schema->type; + bool_schema->type_descriptor.physical_type = tparquet::Type::BOOLEAN; + const auto bool_ranges = run_page_index(std::move(bool_schema), std::string(1, '\x02'), + std::string(1, '\x02'), true); + ASSERT_EQ(bool_ranges.size(), 1); + EXPECT_EQ(bool_ranges[0].start, 0); + EXPECT_EQ(bool_ranges[0].length, 1); +} +TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); -TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("i", arrow::int32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const int32_t inverted_min = 10; - const int32_t inverted_max = 1; - auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( - schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, - 0, true, true, false); - const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], int_stats); - EXPECT_TRUE(converted_int.has_not_null); - EXPECT_FALSE(converted_int.has_min_max); - - const std::string inverted_string_min = "z"; - const std::string inverted_string_max = "a"; - auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( - schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, - false); - const auto converted_string = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - string_stats); - EXPECT_TRUE(converted_string.has_not_null); - EXPECT_FALSE(converted_string.has_min_max); - - auto int_index = - std::make_shared>(inverted_min, inverted_max); - format::parquet::ParquetColumnStatistics page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - int_index, *schema[0], 0, &page_stats)); - EXPECT_TRUE(page_stats.has_not_null); - EXPECT_FALSE(page_stats.has_min_max); - - // These endpoints are inverted within one second. Whole-second validation alone would miss - // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. - constexpr int64_t timestamp_min = 1500000; - constexpr int64_t timestamp_max = 1000000; - auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( - schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, - 0, true, true, false); - auto utc = cctz::utc_time_zone(); - const auto converted_timestamp = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp.has_min_max); - - schema[2]->type = std::make_shared(6); - schema[2]->type_descriptor.doris_type = schema[2]->type; - const auto converted_timestamp_tz = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp_tz.has_min_max); -} + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); -TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), - {double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const double nan = std::numeric_limits::quiet_NaN(); - const double max_value = 2.0; - auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, - true, false); - const auto converted_footer = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], - footer_stats); - auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); - ASSERT_NE(footer_zone_map, nullptr); - EXPECT_TRUE(footer_zone_map->pass_all); - EXPECT_FALSE(footer_zone_map->has_null); - EXPECT_TRUE(footer_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); - - ZoneMapEvalContext footer_ctx; - footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = footer_zone_map}); - Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); - - auto column_index = std::make_shared>(nan, max_value); - format::parquet::ParquetColumnStatistics page_stats; - ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *schema[0], 0, &page_stats)); - auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); - ASSERT_NE(page_zone_map, nullptr); - EXPECT_TRUE(page_zone_map->pass_all); - EXPECT_FALSE(page_zone_map->has_null); - EXPECT_TRUE(page_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); - - ZoneMapEvalContext page_ctx; - page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = page_zone_map}); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + bloom_filter)); } -TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); +TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Max) { + auto column_schema = + std::make_unique(uint32_parquet_bloom_schema()); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct( - std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), - nullptr, &selected, false, &pruning_stats) + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) .ok()); - EXPECT_EQ(selected, std::vector({2})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); - - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1, 2})); - - selected.clear(); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + tparquet::BloomFilterAlgorithm algorithm; + algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm()); + tparquet::BloomFilterHash hash; + hash.__set_XXHASH(tparquet::XxHash()); + tparquet::BloomFilterCompression compression; + compression.__set_UNCOMPRESSED(tparquet::Uncompressed()); + tparquet::BloomFilterHeader bloom_header; + bloom_header.__set_numBytes(static_cast(bloom_filter.size())); + bloom_header.__set_algorithm(algorithm); + bloom_header.__set_hash(hash); + bloom_header.__set_compression(compression); + std::vector bloom_bytes; + ThriftSerializer serializer(/*compact=*/true, 64); + ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok()); + bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(), + bloom_filter.data() + bloom_filter.size()); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(1); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_bloom_filter_offset(0); + column_metadata.__set_bloom_filter_length(static_cast(bloom_bytes.size())); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_version(1); + metadata.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + + format::parquet::ParquetFileContext file_context; + file_context.native_file = std::make_shared(std::move(bloom_bytes)); + auto request = request_with_bloom_conjunct( + column_schema->type, + {Field::create_field(static_cast(present_value))}); + std::vector> schema; + schema.push_back(std::move(column_schema)); + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NULL)), - nullptr, &selected, false, &pruning_stats) + metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0})); + EXPECT_EQ(selected_row_groups, std::vector({0})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } -TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, false); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, EmptyDictionaryRowGroupIsSkippedBeforeMetadataProbes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::BYTE_ARRAY); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(0); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_dictionary_page_offset(0); + column_metadata.__set_encodings({tparquet::Encoding::RLE_DICTIONARY}); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(0); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(0); + thrift_metadata.__set_row_groups({row_group}); + + format::parquet::NativeParquetMetadata native_metadata(thrift_metadata, 0); + ASSERT_TRUE(native_metadata.init_schema(false, false).ok()); + format::parquet::ParquetFileContext file_context; + file_context.native_file = + std::make_shared(std::vector {}); + file_context.native_metadata = &native_metadata; + + auto column_schema = std::make_unique(); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::BYTE_ARRAY; + column_schema->type_descriptor.is_string_like = true; + std::vector> schema; + schema.push_back(std::move(column_schema)); - std::vector selected; + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared(std::make_shared())}; + std::vector selected_row_groups; format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); - - auto plain_reader = make_reader(table, 2, false, false); - auto plain_schema = build_file_schema(*plain_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *plain_reader->metadata(), plain_reader.get(), plain_schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) + thrift_metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_TRUE(selected_row_groups.empty()); } -TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, true); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, InvalidCandidateRowGroupReturnsCorruption) { + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + format::FileScanRequest request; + const std::vector> schema; + const std::vector candidates {1}; + std::vector selected_row_groups; + + const auto status = format::parquet::select_row_groups_by_metadata( + metadata, schema, request, &candidates, &selected_row_groups, false, nullptr, nullptr, + nullptr, nullptr); + EXPECT_TRUE(status.is()) << status; +} +TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { + format::parquet::ParquetTypeDescriptor binary_type; + binary_type.physical_type = tparquet::Type::BYTE_ARRAY; + + tparquet::Statistics max_only; + max_only.__set_max("III"); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only, false)); + + tparquet::Statistics legacy_different; + legacy_different.__set_min("III"); + legacy_different.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, + legacy_different, false)); + + tparquet::Statistics legacy_equal; + legacy_equal.__set_min("same"); + legacy_equal.__set_max("same"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal, + false)); + + tparquet::Statistics type_defined; + type_defined.__set_min_value("III"); + type_defined.__set_max_value("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + false)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + true)); + + tparquet::Statistics mixed_fields; + mixed_fields.__set_min_value("III"); + mixed_fields.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields, + true)); +} + +TEST(NativeParquetStatisticsTest, ExplicitlyInexactNumericBoundsCannotBackMinMaxAggregate) { + format::parquet::ParquetTypeDescriptor int32_type; + int32_type.physical_type = tparquet::Type::INT32; + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(0)); + statistics.__set_max_value(encode_int32(100)); + EXPECT_TRUE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(false); + statistics.__set_is_max_value_exact(true); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(true); + statistics.__set_is_max_value_exact(false); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); +} + +TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder) { + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(1)); + statistics.__set_max_value(encode_int32(2)); + statistics.__set_null_count(0); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(1); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(100))}; + std::vector selected_row_groups; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - auto no_stats_reader = make_reader(table, 2, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + EXPECT_EQ(selected_row_groups, std::vector({0})); + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({encode_int32(1)}); + page_index.column_index.__set_max_values({encode_int32(2)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, page_index); + std::vector selected_ranges; + std::map skip_plans; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); -} - -TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { - auto input = arrow::io::ReadableFile::Open( - "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); - ASSERT_TRUE(input.ok()); - auto reader = ::parquet::ParquetFileReader::Open(*input); - ASSERT_EQ(reader->metadata()->num_row_groups(), 2); - auto& bloom_filter_reader = reader->GetBloomFilterReader(); - for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { - auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); - ASSERT_NE(row_group_reader, nullptr); - ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); - } - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - auto request = request_with_bloom_conjunct(std::make_shared(), - {Field::create_field(12345)}); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - false, &pruning_stats) + EXPECT_EQ(selected_ranges.size(), 1); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + metadata.__set_column_orders({order}); + selected_row_groups.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - true, &pruning_stats) + EXPECT_TRUE(selected_row_groups.empty()); + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); -} - -TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(4)}), - ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(3)}), - ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { - auto bool_type = std::make_shared(); - auto bool_filter = - bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); - auto bool_ctx = bloom_context(bool_type, bool_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(false)}), - bool_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(true)}), - bool_ctx), - ZoneMapFilterResult::kMayMatch); - - auto string_type = std::make_shared(); - auto string_filter = bloom_filter_for_fields( - {Field::create_field("alpha"), Field::create_field("omega")}, - TYPE_STRING); - auto string_ctx = bloom_context(string_type, string_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("beta")}), - string_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("alpha")}), - string_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { - auto int_type = std::make_shared(); - BloomFilterEvalContext missing_ctx; - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), - ZoneMapFilterResult::kMayMatch); - - auto smallint_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); - auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(smallint_type, {Field::create_field(2)}), - unsupported_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { - const auto column_schema = uint32_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - const uint32_t present_value = 4000000000U; - int32_t physical_value; - memcpy(&physical_value, &present_value, sizeof(physical_value)); - bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); - - // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical - // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical - // bit pattern before probing the file bloom filter. - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field( - static_cast(present_value))}), - *bloom_filter)); - - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts( - column_schema.type, - {Field::create_field( - static_cast(std::numeric_limits::max()) + 1)}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { - const auto column_schema = fixed_len_string_parquet_bloom_schema(4); - auto bloom_filter = parquet_bloom_filter(); - - const std::string present_value = "abcd"; - ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); - bloom_filter->InsertHash( - bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { - const auto column_schema = float16_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), - *bloom_filter)); + EXPECT_TRUE(selected_ranges.empty()); +} + +TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) { + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(0))}; + + for (const int64_t contradictory_null_count : {5, 11}) { + SCOPED_TRACE(contradictory_null_count); + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_null_pages({true}); + page_index.column_index.__set_null_counts({contradictory_null_count}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 10, &selected_ranges, + &skip_plans, nullptr) + .ok()); + // ColumnIndex is optional. An impossible all-null claim must fall back to reading the + // ten-row data page instead of proving that no value can satisfy the predicate. + ASSERT_EQ(selected_ranges.size(), 1); + EXPECT_EQ(selected_ranges[0].start, 0); + EXPECT_EQ(selected_ranges[0].length, 10); + } } } // namespace diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index 4bca77c1803b49..0680cc828a9f7a 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -17,478 +17,49 @@ #include "format_v2/parquet/parquet_type.h" -#include -#include #include -#include -#include -#include - -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" namespace doris::format::parquet { -namespace { - -::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - return descriptor; -} - -ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { - auto descriptor = make_descriptor(node); - return resolve_parquet_type(descriptor.Column(0)); -} - -PrimitiveType primitive_type(const DataTypePtr& type) { - return remove_nullable(type)->get_primitive_type(); -} - -int scale_of(const DataTypePtr& type) { - return remove_nullable(type)->get_scale(); -} - -std::shared_ptr make_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.Append(0x3E00).ok()); - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetTypeDescriptor resolve_arrow_float16_type() { - const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); - const auto table = arrow::Table::Make(schema, {make_float16_array()}); - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - - auto reader = ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); - return resolve_parquet_type(reader->metadata()->schema()->Column(0)); -} - -} // namespace - -TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { - struct Case { - int bit_width; - bool is_signed; - PrimitiveType expected_type; - bool expected_unsigned; - }; - const std::vector cases = { - {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, - {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, - {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, - {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.bit_width); - const auto node = ::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), - test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); - const auto type = resolve_node(node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_TRUE(type.supports_record_reader); - } -} - -TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { - const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - ASSERT_NE(time_millis.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); - EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); - - const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_us", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64)); - ASSERT_NE(time_micros.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - - const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_adjusted", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.supports_record_reader); - EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); - - const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ns", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(timestamp_nanos.doris_type, nullptr); - EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(timestamp_nanos.is_timestamp); - EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); - EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); - EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); -} - -TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { - struct Case { - ::parquet::LogicalType::TimeUnit::unit parquet_unit; - bool adjusted_to_utc; - ParquetTimeUnit expected_unit; - ParquetExtraTypeInfo expected_extra; - int expected_scale; - }; - const std::vector cases = { - {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_scale); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, - test_case.parquet_unit), - ::parquet::Type::INT64)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); - EXPECT_TRUE(type.is_timestamp); - EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); - EXPECT_EQ(type.time_unit, test_case.expected_unit); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { - const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS)); - EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.supports_record_reader); - EXPECT_FALSE(converted_time.unsupported_reason.empty()); - - const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MILLIS)); - ASSERT_NE(converted_timestamp.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(converted_timestamp.is_timestamp); - EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); - const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MICROS)); - ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); - EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); - EXPECT_TRUE(converted_timestamp_micros.is_timestamp); - EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); -} - -TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { - struct Case { - ::parquet::ConvertedType::type converted_type; - ::parquet::Type::type physical_type; - PrimitiveType expected_type; - int bit_width; - bool expected_unsigned; - DecodedValueKind expected_value_kind; - }; - const std::vector cases = { - {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, - DecodedValueKind::UINT32}, - {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, - DecodedValueKind::INT64}, - {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, - DecodedValueKind::UINT64}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.converted_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, - test_case.converted_type)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); - } -} - -TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { - struct Case { - ::parquet::Type::type physical_type; - int type_length; - int precision; - int scale; - PrimitiveType expected_type; - ParquetExtraTypeInfo expected_extra; - }; - const std::vector cases = { - {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, - {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, - ParquetExtraTypeInfo::DECIMAL_INT64}, - {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.physical_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, - ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, - test_case.scale)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_TRUE(type.is_decimal); - EXPECT_FALSE(type.is_string_like); - EXPECT_EQ(type.decimal_precision, test_case.precision); - EXPECT_EQ(type.decimal_scale, test_case.scale); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { - const std::vector> string_like_logical_types = { - ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), - ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; - for (const auto& logical_type : string_like_logical_types) { - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); - EXPECT_TRUE(type.is_string_like); - } - - const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( - "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(uuid.doris_type, nullptr); - EXPECT_TRUE(uuid.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); - EXPECT_TRUE(uuid.is_string_like); - - const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), - ::parquet::Type::INT32)); - ASSERT_NE(date.doris_type, nullptr); - EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); - - const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), - ::parquet::Type::INT64)); - ASSERT_NE(decimal64.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); - EXPECT_TRUE(decimal64.is_decimal); - EXPECT_EQ(decimal64.decimal_precision, 18); - EXPECT_EQ(decimal64.decimal_scale, 6); - EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); - - const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(decimal128.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); - EXPECT_TRUE(decimal128.is_decimal); - EXPECT_EQ(decimal128.decimal_precision, 38); - EXPECT_EQ(decimal128.decimal_scale, 6); - EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - EXPECT_FALSE(decimal256.is_string_like); -} - -TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { - const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), - ::parquet::Type::INT32)); - ASSERT_NE(logical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(logical_type.integer_bit_width, 8); - - const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8)); - ASSERT_NE(converted_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(converted_type.integer_bit_width, 8); - - const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); - ASSERT_NE(physical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); - EXPECT_EQ(physical_type.integer_bit_width, -1); -} - -TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_FALSE(decimal256.is_string_like); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(plain_binary.doris_type, nullptr); - EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); - EXPECT_TRUE(plain_binary.is_string_like); - - const auto float16 = resolve_arrow_float16_type(); - ASSERT_NE(float16.doris_type, nullptr); - EXPECT_TRUE(float16.doris_type->is_nullable()); - EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(float16.fixed_length, 2); - EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); - EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(float16.is_string_like); - EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); -} - -TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { - const auto null_type = resolve_parquet_type(nullptr); - EXPECT_EQ(null_type.doris_type, nullptr); - EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - EXPECT_TRUE(null_type.supports_record_reader); - - const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); - ASSERT_NE(int96.doris_type, nullptr); - EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); -} - -TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { - struct Case { - ::parquet::schema::NodePtr node; - PrimitiveType expected_type; - DecodedValueKind expected_kind; - bool expected_string_like = false; - }; - const std::vector cases = { - {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BOOLEAN), - TYPE_BOOLEAN, DecodedValueKind::BOOL}, - {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - TYPE_INT, DecodedValueKind::INT32}, - {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT64), - TYPE_BIGINT, DecodedValueKind::INT64}, - {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FLOAT), - TYPE_FLOAT, DecodedValueKind::FLOAT}, - {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - TYPE_DOUBLE, DecodedValueKind::DOUBLE}, - {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY), - TYPE_STRING, DecodedValueKind::BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::NONE, 4), - TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT96), - TYPE_DATETIMEV2, DecodedValueKind::INT96}, +TEST(ParquetTypeTest, DecodedValueKindUsesNativePhysicalTypes) { + ParquetTypeDescriptor descriptor; + + const std::pair cases[] = { + {tparquet::Type::BOOLEAN, DecodedValueKind::BOOL}, + {tparquet::Type::INT32, DecodedValueKind::INT32}, + {tparquet::Type::INT64, DecodedValueKind::INT64}, + {tparquet::Type::INT96, DecodedValueKind::INT96}, + {tparquet::Type::FLOAT, DecodedValueKind::FLOAT}, + {tparquet::Type::DOUBLE, DecodedValueKind::DOUBLE}, + {tparquet::Type::BYTE_ARRAY, DecodedValueKind::BINARY}, + {tparquet::Type::FIXED_LEN_BYTE_ARRAY, DecodedValueKind::FIXED_BINARY}, }; - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_type); - const auto type = resolve_node(test_case.node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); - EXPECT_EQ(type.is_string_like, test_case.expected_string_like); - EXPECT_TRUE(type.supports_record_reader); + for (const auto& [physical_type, expected] : cases) { + descriptor.physical_type = physical_type; + descriptor.is_unsigned_integer = false; + descriptor.integer_bit_width = -1; + EXPECT_EQ(decoded_value_kind(descriptor), expected); } } -TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { - EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); +TEST(ParquetTypeTest, DecodedValueKindPreservesUnsignedWidth) { + ParquetTypeDescriptor descriptor; + descriptor.is_unsigned_integer = true; - const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(nanos_time.doris_type, nullptr); - EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); - EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 32; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT32); - const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns_utc", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); - EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT64; + descriptor.integer_bit_width = 64; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT64); - EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Float16(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), - ::parquet::ParquetException); + // Narrow unsigned logical integers still use the signed physical decoder; conversion + // happens after decoding so the on-disk bit width remains the single source of truth. + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 16; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::INT32); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hive_reader_test.cpp b/be/test/format_v2/table/hive_reader_test.cpp index d4f9be81129c6c..98d39fbd328093 100644 --- a/be/test/format_v2/table/hive_reader_test.cpp +++ b/be/test/format_v2/table/hive_reader_test.cpp @@ -79,7 +79,7 @@ bool has_name_mapping(const ColumnDefinition& column, const std::string& name) { class HiveV2ReaderTest : public testing::Test { public: - HiveV2ReaderTest() : profile("hive_v2_reader_test") { state.set_query_options(query_options); } + HiveV2ReaderTest() : state(query_options, query_globals), profile("hive_v2_reader_test") {} protected: TQueryOptions query_options; diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 2dd001469d8a66..6f9bea89ffe943 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -23,10 +23,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -122,6 +124,23 @@ TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { return table_format_params; } +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -300,5 +319,47 @@ TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader std::filesystem::remove_all(test_dir); } +TEST(HudiHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + verify_first_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..a219ba37f3dd8d --- /dev/null +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { +namespace { + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPositionReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("position_delete_system_table_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + delete_file.__set_path("/not-opened-during-prepare.parquet"); + + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(delete_file); + ASSERT_TRUE(reader._init_split().ok()); + + ASSERT_NE(reader._position_reader_profile, nullptr); + EXPECT_NE(reader._position_reader_profile, &profile); + EXPECT_EQ(profile.get_child("IcebergPositionDeleteFileReader"), + reader._position_reader_profile); +} + +} // namespace +} // namespace doris::format::iceberg diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index b7af62c400251d..782463e335a743 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -25,12 +25,15 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -57,13 +60,13 @@ #include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" #include "exec/scan/access_path_parser.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" -#include "format_v2/deletion_vector_reader.h" +#include "format/table/deletion_vector_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" @@ -275,6 +278,19 @@ std::shared_ptr build_nullable_int64_array( return finish_array(&builder); } +std::shared_ptr build_nullable_string_array( + const std::vector>& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + if (value.has_value()) { + EXPECT_TRUE(builder.Append(*value).ok()); + } else { + EXPECT_TRUE(builder.AppendNull().ok()); + } + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -735,6 +751,31 @@ void write_position_delete_parquet_file(const std::string& file_path, builder.build())); } +void write_nullable_position_delete_parquet_file( + const std::string& file_path, + const std::vector>& data_file_paths, + const std::vector>& positions) { + ASSERT_EQ(data_file_paths.size(), positions.size()); + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), true), + arrow::field("pos", arrow::int64(), true), + }); + auto table = arrow::Table::Make(schema, {build_nullable_string_array(data_file_paths), + build_nullable_int64_array(positions)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(positions.size()), + builder.build())); +} + int64_t write_iceberg_deletion_vector_file(const std::string& file_path, const std::vector& deleted_positions) { roaring::Roaring64Map rows; @@ -768,6 +809,12 @@ class ScopedDebugPoint { DebugPoints::instance()->add(_name); } + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + ~ScopedDebugPoint() { DebugPoints::instance()->remove(_name); config::enable_debug_points = _enable_debug_points; @@ -2411,8 +2458,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default( - "\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2422,9 +2469,11 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( 100, {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "Ej5FZ+ibEtOkVkJmFBdAAA==", - external_primitive_type(TPrimitiveType::VARBINARY, 16), - true)})}); + external_schema_field( + "added_binary", 1, {}, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + external_primitive_type(TPrimitiveType::VARBINARY, + static_cast(binary_default.size())), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -3405,7 +3454,25 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { first_split.cache = &cache; first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); - ASSERT_TRUE(reader.prepare_split(first_split).ok()); + const auto* total_timer = profile.get_counter("TableReader"); + const auto* prepare_timer = profile.get_counter("PrepareSplitTime"); + ASSERT_NE(total_timer, nullptr); + ASSERT_NE(prepare_timer, nullptr); + const int64_t total_before = total_timer->value(); + const int64_t prepare_before = prepare_timer->value(); + constexpr int64_t DELETE_FILE_DELAY_NS = 8'000'000; + { + ScopedDebugPoint delay_delete_file_scan( + "IcebergTableReader.prepare_split.before_delete_file_scan", + [] { std::this_thread::sleep_for(std::chrono::milliseconds(8)); }); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + } + const int64_t total_delta = total_timer->value() - total_before; + const int64_t prepare_delta = prepare_timer->value() - prepare_before; + // A cache-miss delete-file scan is derived prepare work; both common parent timers must + // contain it so the expensive miss cannot disappear between profile levels. + EXPECT_GE(prepare_delta, DELETE_FILE_DELAY_NS); + EXPECT_GE(total_delta, DELETE_FILE_DELAY_NS); EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); // The cached delete file contains entries for every referenced data file, so another split can @@ -3468,6 +3535,147 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path}, + {int64_t {1}, int64_t {3}}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeleteFileWithActualNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_actual_null_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, std::nullopt}, + {int64_t {1}, int64_t {2}}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + auto status = reader.prepare_split(split_options); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("file_path contains null values"), std::string::npos) + << status.to_string(); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeletePosWithActualNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_pos_null_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path}, + {int64_t {1}, std::nullopt}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + auto status = reader.prepare_split(split_options); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("pos contains null values"), std::string::npos) + << status.to_string(); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergTableReaderIgnoresPositionDeleteFilesWhenDeletionVectorPresent) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_delete_files_merge_test"; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 08566f9253fcda..3c85eb4632f86c 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include "core/assert_cast.h" @@ -44,9 +46,9 @@ #include "core/field.h" #include "exec/common/endian.h" #include "format/format_common.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" #include "format_v2/column_data.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/jni/paimon_jni_reader.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -60,6 +62,23 @@ namespace doris::format { namespace { +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -763,6 +782,49 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + verify_first_split(FileFormat::JNI, make_paimon_jni_range()); +} + TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index ddc52f41cdc5e0..48b38d78e19662 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -50,10 +50,10 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_varbinary.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/table/iceberg_scan_semantics.h" #include "gen_cpp/Exprs_types.h" @@ -265,8 +265,7 @@ VExprSPtr runtime_filter_wrapper_expr(VExprSPtr impl) { node.__set_node_type(TExprNodeType::SLOT_REF); node.__set_type(std::make_shared()->to_thrift()); node.__set_num_children(1); - return VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0, false, - /*filter_id=*/1); + return RuntimeFilterExpr::create_shared(node, std::move(impl), 0, false, /*filter_id=*/1); } class NonDeterministicPartitionPredicate final : public VExpr { @@ -331,9 +330,50 @@ class NullableArrayBigintDefaultExpr final : public VExpr { class TableReaderMaterializeTestHelper final : public TableReader { public: + using TableReader::_materialize_mapping_column; using TableReader::_materialize_map_mapping_column; }; +TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) { + auto keys = ColumnString::create(); + keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20); + auto values = ColumnString::create(); + values->insert_data(std::string(1UL << 20, 'v').data(), 1UL << 20); + const auto* original_value_bytes = values->get_chars().data(); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); + auto null_map = ColumnUInt8::create(1, 0); + ColumnPtr source = ColumnNullable::create(std::move(map), std::move(null_map)); + + const auto string_type = std::make_shared(); + const auto map_type = make_nullable(std::make_shared(string_type, string_type)); + Block block; + block.insert({source, map_type, "large_map"}); + source.reset(); + + ColumnMapping mapping; + mapping.global_index = GlobalIndex(0); + mapping.table_column_name = "large_map"; + mapping.file_column_name = "large_map"; + mapping.file_local_id = 0; + mapping.file_type = map_type; + mapping.table_type = map_type; + mapping.is_trivial = true; + mapping.projection = + VExprContext::create_shared(VSlotRef::create_shared(0, 0, -1, map_type, "large_map")); + TableReaderMaterializeTestHelper reader; + ColumnPtr detached; + ASSERT_TRUE(reader._materialize_mapping_column(mapping, &block, 1, &detached, + /*take_projection_result=*/true) + .ok()); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + const auto& detached_nullable = assert_cast(*detached); + const auto& detached_map = assert_cast(detached_nullable.get_nested_column()); + const auto& detached_values = assert_cast(detached_map.get_values()); + EXPECT_EQ(detached_values.get_chars().data(), original_value_bytes); +} + VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, int right_column_id) { const auto int_type = std::make_shared(); @@ -1347,6 +1387,43 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_FALSE(predicate_executed); + ASSERT_TRUE(reader.close().ok()); +} + TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); @@ -1767,6 +1844,7 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_FALSE(eos); EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(fake_state->close_count, 1); EXPECT_TRUE(reader.current_split_uses_metadata_count()); @@ -1778,6 +1856,58 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { EXPECT_EQ(fake_state->last_aggregate_request->columns[0].projection.local_id(), 0); } +TEST(TableReaderTest, PushDownCountEmitsAtMostOneRuntimeBatch) { + const auto nullable_bigint_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_bigint_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); + set_name_identifiers(&projected_columns); + + TQueryOptions query_options; + query_options.__set_batch_size(2); + RuntimeState state {query_options, TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 5; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + bool eos = false; + for (const size_t expected_rows : {2, 2, 1}) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), expected_rows); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + } + + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->open_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(reader.close().ok()); +} + TEST(TableReaderTest, PushDownCountStarIgnoresProjectedPlaceholderColumn) { const auto nullable_int_type = make_nullable(std::make_shared()); std::vector file_schema; @@ -1816,6 +1946,7 @@ TEST(TableReaderTest, PushDownCountStarIgnoresProjectedPlaceholderColumn) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_FALSE(eos); EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); ASSERT_TRUE(fake_state->last_request != nullptr); ASSERT_EQ(fake_state->last_request->count_star_placeholder_columns.size(), 1); EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); @@ -4249,6 +4380,7 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + EXPECT_TRUE(file_request.predicate_only_columns.empty()); ASSERT_EQ(file_request.local_positions.size(), 2); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); @@ -4765,49 +4897,6 @@ TEST(TableReaderTest, ProjectedPartitionColumnUsesSplitPartitionValue) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedNullPartitionColumnPreservesNull) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_table_reader_null_partition_value_test"; - std::filesystem::remove_all(test_dir); - std::filesystem::create_directories(test_dir); - - const auto file_path = (test_dir / "split.parquet").string(); - write_parquet_file(file_path, 1, "one"); - - std::vector projected_columns; - auto partition_column = make_table_column(1, "value", std::make_shared()); - partition_column.is_partition_key = true; - projected_columns.push_back(std::move(partition_column)); - - RuntimeState state {TQueryOptions(), TQueryGlobals()}; - set_name_identifiers(&projected_columns); - TableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = nullptr, - .io_ctx = nullptr, - .runtime_state = &state, - .scanner_profile = nullptr, - }) - .ok()); - - auto split_options = build_split_options(file_path); - split_options.partition_values.emplace("value", Field::create_field(Null())); - ASSERT_TRUE(reader.prepare_split(split_options).ok()); - - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - ASSERT_FALSE(eos); - - expect_nullable_column_all_null(*block.get_by_position(0).column); - - ASSERT_TRUE(reader.close().ok()); - std::filesystem::remove_all(test_dir); -} - TEST(TableReaderTest, ConstantPartitionFilterSkipsSplitWhenFalse) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_constant_partition_filter_skip_test"; diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b05ea798ee46b0..82a3ba03667753 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -99,14 +99,135 @@ format-specific checklist when reviewing Parquet or ORC. - Keep index construction, predicate translation, cache lookup, and virtual-column setup out of per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema traversal, expression cloning, metadata parsing, allocation, and conversion. +- Keep Profile counters in the visible `FileScannerV2 -> TableReader -> FileReader -> IO` hierarchy. + Format-specific readers, such as `ParquetReader`, belong below `FileReader`; lifecycle, metadata, + index, predicate, decode, materialization, and physical I/O paths must all have timers at the layer + that owns the work. Flush recursively aggregated child-reader statistics at every batch boundary, + including empty-selection and error exits, so a slow in-progress scan is diagnosable before close. - Require format readers to populate the common `ReaderStatistics` accurately where applicable: filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and Bloom read time. +- Reject dead or ambiguous counters. In particular, `FilteredBytes` counts compressed bytes of + projected physical chunks avoided by pruning, not every child in the Row Group; footer read, + footer parse, lazy page-index materialization, and page-index predicate evaluation need distinct + timers. Raw I/O counters stay under `IO` even when a format reader initiates the request. - Evaluate performance with representative format versions, writers, data ordering, predicate selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. +## Parquet Native Decode Boundary + +- V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the + v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Footer parsing, schema-ID assignment, and the cached native metadata tree must also be v2-owned. + Reusing a stable base file identity is allowed, but require a v2 cache type discriminator so + v1/v2 metadata objects can never be cross-cast. Production planning must not retain or rebuild an + Arrow `FileMetaData` tree from the serialized footer. +- Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. + Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, + builders, `DecodedColumnView`, or another decoded leaf batch. +- Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, + decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and + fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. + Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, + statistics conversion, and min/max pruning must use the same timezone rule. +- Route plain, dictionary, and decoded timestamp inputs through one checked conversion contract. + Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, + and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same + strict/non-strict and dictionary-ID propagation rules as other direct types. +- Verify schema-change routing separately from physical decode. The reader must emit the projected + file type, while `ColumnMapper`/`TableReader` perform file-to-table casts after file predicates. + Any different requested type at the native reader boundary must fail as an invariant violation; + do not add a reader-local conversion column or decoder-facing conversion ABI. +- Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload + ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate + every index before access, and invalidate cached dictionary state at Row Group/file/type changes. + Dictionary-entry predicate evaluation and later row-value flattening must reuse that same typed + generation rather than serializing, parsing, or converting the dictionary twice. +- Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded + values without allocating output; null runs must append defaults without advancing payload. +- For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the + whole Column Chunk uses compatible PLAIN value pages and every Expr advertises raw fixed-value + evaluation with identical Doris comparison semantics, including NaN ordering. The fallback + decision must precede definition-level consumption. Disable the direct path when a residual or + delete conjunct still references the hidden slot, because it needs the materialized payload. + Verify sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, residual slot reuse, and a stable row-shaped placeholder. +- For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode + dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, + decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review + blocker for a scalar destination that supports in-place expansion. Selection ranges belong to + persistent reader scratch; per-range virtual SerDe/decoder calls in the hot path are a review + blocker. Fixed PLAIN should + bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate + every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and + compact. Any remaining NULL-interleaving fallback must preserve logical output order without a + decoded intermediate column and be counted by `HybridSelectionNullFallbackBatches`. +- Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, + null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct + without materializing an intermediate complex column. +- For MAP, the non-null key leaf owns the outer entry shape. Validate the materialized key/value + entry counts, but do not compare their raw repetition vectors: a nested value legitimately has + deeper levels. For STRUCT, compare each sibling only at the current parent boundary and ignore + repetition owned by a deeper child collection. +- Require a bounded high-water policy for persistent definition/repetition, null, selection, + conversion, dictionary-index, and decoder-owned scratch. Distinguish active bytes from retained + capacity: never release an oversized buffer while the current batch still needs it, and require + three ordinary/idle batches before releasing capacity above the high-water limit. Test both + outlier release and steady-state reuse so the policy does not create allocation thrash. +- Review all decoder read and skip paths as equally exposed corruption boundaries. Check requested + counts against remaining/declared values before pointer arithmetic or narrowing; use checked + addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before + allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed + value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Validate every decoded definition/repetition level against the schema maximum in batch, run, and + single-value cursor APIs. Page value counts must not drive eager nested scratch allocation; + reserve from the requested parent-row frontier and cover tiny-payload huge-count Page V1/V2 files. +- Before Snappy decompression, inspect the encoded uncompressed length and validate destination + capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; + an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. +- For a STRUCT whose projected children are all missing after schema evolution, require a + levels-only physical reference leaf. It must advance and validate encoded payload cursors while + deriving the synthetic child count, without constructing a discarded string/complex column. +- `CountColumnReader` must use the native levels-only reader and must not decode payload or call + Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, + value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, + filtered-value skips, and page fragmentation. +- Validate every signed Column Chunk offset/length before converting it to `size_t`. The dictionary + offset is usable only when it is non-negative and precedes the data offset; the complete range + must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it + at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. +- Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, + the first physical location to equal the owning ColumnMetaData `data_page_offset`, strictly + increasing row ordinals and physical offsets, positive sizes, non-overlapping page ranges, and + containment in the owning Column Chunk. Discard a malformed index before selecting the indexed + reader. +- Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. + Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. +- Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, + and COUNT paths. Pre-Arrow-3 parquet-cpp Data Page V2 payloads remain compressed despite the + historical `is_compressed=false` flag. +- Preserve nullable conversion semantics in direct native materialization. Numeric, DATE, + DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL + only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow + the selected dictionary IDs to output rows. +- Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target + precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion + failure; plain and dictionary integer/binary paths must narrow only afterward. +- For cold small-file tests, separate footer I/O/Thrift parse from native schema and index planning. + V2 must not retain serialized footer bytes after the native metadata tree is initialized; v1 opens + remain independent. +- For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from + byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range + servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm + scans must not depend on the footer read having populated an incidental transport buffer. +- Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the + dense byte span once and synthesize offsets; validate this execution contract without flaky + wall-clock assertions. + ## Parquet Multi-Level Filtering - Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed @@ -129,12 +250,37 @@ format-specific checklist when reviewing Parquet or ORC. - Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows while advancing all readers correctly. Predicate columns should be read/prefetched first; output prefetch should wait for survivors when filtering is active. +- For safe staged single-column predicates, review the observed cost/rejection ordering and its + cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may + stop at a low-probability reach prefix, while output-column prefetch may start early only after a + learned high survival ratio. Static conjunct schedules and position maps belong to the scanner + lifecycle; after eight warm-up samples, per-predicate clocks should be sampled only periodically. + Cache the batch SelectionVector's dense bitmap by generation so a wide lazy projection does not + rebuild the same O(batch) filter for every column. +- Single-column predicate rounds may keep previously read columns in their original row mappings. + Require one alignment compaction before multi-column, delete, or output boundaries, and expose + its time, bytes, and count instead of charging repeated movement invisibly to predicate time. + Reused compaction masks must be fully cleared when coordinate-space sizes shrink. +- PLAIN BYTE_ARRAY must not parse lengths in both decoder and destination column. Review the direct + payload-offset/cumulative-offset contract, uint32 overflow checks, surviving-span coverage, and + the legacy consumer fallback for non-string logical types. +- Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. + Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; a production Arrow + metadata adapter or rebuilt `FileMetaData` tree is a review blocker. +- Fixed-width conversion fast paths may remove row branches only when the source domain provably + fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict + NULL marking must retain corrupt-value tests. +- Recursive reader-profile publication and retained-scratch inspection should be amortized, but Row + Group reset/EOF/close must force the final profile delta before reader destruction. +- Accumulated lazy-column skips must not allocate one dense byte per rejected prefix row. Require a + fixed chunk bound (including multiple lazy columns) or a level-only/all-filtered cursor contract. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. - Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped - rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and - remote I/O. + rows/pages, raw and filtered rows, dictionary-row filtering, PLAIN direct-predicate batches/rows, + lazy-read savings, cache sources, and remote I/O. - Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md index 66b96de1204d14..b2fe43ae76f6b0 100644 --- a/docs/file-scanner-v2-design.md +++ b/docs/file-scanner-v2-design.md @@ -282,6 +282,23 @@ Scan optimization remains maintainable only when costs are visible, sources are and failure semantics are explicit. V2 provides three complementary views: Query Profile, query resource context, and global metrics. +Query Profile uses one stable ownership tree: + +```text +FileScannerV2 +└── TableReader + └── FileReader + ├── format-specific reader (ParquetReader, OrcReader, ...) + └── IO +``` + +Scanner lifecycle and Split scheduling are charged to `FileScannerV2`; table-schema restoration, +delete handling, and reader lifecycle are charged to `TableReader`; metadata, index, decode, and +materialization are charged to `FileReader` and its format subtree; physical reads, bytes, cache +waits, and remote/local attribution remain under `IO`. Every executable lifecycle path needs a +timer, and cumulative child-reader statistics must be published at batch boundaries as well as +close, so an active slow query never presents an unexplained timing gap. + ```mermaid flowchart LR R[FileReader and FileCache Raw Statistics] --> P[Query Profile] diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 63f63819670dd2..70c041a89ccd48 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -25,8 +25,9 @@ predicate columns for surviving ranges, and finally defer output-column reads un - **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small I/O solve different problems and are not interchangeable. -**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It -does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. +**Scope:** This document covers the FileScannerV2 Parquet Reader pipeline, the native page/decode +kernel, selection-aware materialization, and complex-type reconstruction contracts. Expression +implementation details remain outside its scope. ## 2. Overall Architecture @@ -40,9 +41,10 @@ flowchart TB B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] - E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] - F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] - G --> H[Doris FileReader / FileCache / Remote FS] + E --> F[ParquetColumnReader
Persistent Column State and Complex Reconstruction] + F --> G[Native ColumnChunk / Page / Encoding Decoders
Selection-aware Doris Materialization] + G --> H[ParquetFileContext / Stream Adapter
Page Cache, MergeRange, Prefetch] + H --> I[Doris FileReader / FileCache / Remote FS] ``` | Layer | Core responsibilities | Responsibilities intentionally excluded | @@ -58,6 +60,64 @@ flowchart TB > layer can use footer, page index, dictionary, and other format knowledge while upper layers retain > uniform scan semantics. +### 2.1 Native Reader Target and Migration State + +The target execution path does not use Arrow builders or `ReadRecords` to decode data pages. Doris +owns the Column Chunk, page, level, encoding, selection, conversion, and materialization state, and +writes directly into Doris columns. This follows the same broad separation used by DuckDB: metadata +planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow +cursor-based contracts rather than an Arrow array as an intermediate result. + +V1 remains the differential baseline, but v2 owns an independent reader implementation. Its page +and encoding algorithms started from proven Doris behavior and are maintained under the v2 tree; +the production v2 path never instantiates the v1 `ParquetColumnReader`: + +| Stage | State and boundary | +| --- | --- | +| Native encoding kernel | V2 owns the Column Chunk, page, level, and encoding decoders under `be/src/format_v2/parquet/reader/native/`. Decoders expose raw fixed/binary spans, validated dictionary indices, and skip operations; they never write Doris columns. | +| Logical materialization | `DataTypeSerDe` interprets Parquet physical/logical metadata and writes decoded spans directly into the final Doris column. Typed dictionaries and scratch are persistent per leaf reader. | +| Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | +| Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | +| Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | +| Compatibility removal | Production v2 has no Arrow data-read or metadata adapter and never falls back to the v1 reader; unsupported combinations return explicit errors. Arrow remains only in test oracles and fixture writers. | + +Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or +builders. V2 parses and owns the Thrift footer, schema parser, field-ID assignment, physical schema, +statistics, dictionary metadata, bloom filters, and page indexes. Production planning never builds +an Arrow metadata tree; Arrow is limited to fixture writers and differential test oracles. + +All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness +and performance control. Compatibility is demonstrated through differential tests and the explicit +type/encoding matrix, not through a runtime dependency on v1 code. + +### 2.2 Native Interface Ownership + +```mermaid +flowchart LR + A[Native Footer / Physical Schema] --> B[RowGroupReadPlan] + B --> C[Persistent ParquetColumnReader] + C --> D[Persistent ColumnChunkReader] + D --> E[Page Reader + Level Decoders] + E --> F[Encoding Decoder] + F --> G[ColumnSelectVector] + G --> H[Doris MutableColumn] + C --> I[Shared Complex Level Plan] + I --> G +``` + +- **Physical schema contract:** Immutable physical type, fixed length, maximum definition and + repetition levels, and repeated-parent definition threshold. It owns no Arrow descriptor and + borrows no temporary table-schema object. +- **Persistent column state:** Page/decompress buffers, dictionary decoder, SerDe/conversion state, + null maps, selection ranges, binary-value references, and builder capacity live with the column + reader and are logically reset rather than recreated for each batch. +- **Decoder contract:** Consume a known number of logical level entries and encoded payload values, + then materialize only selected rows. Decoders do not decide table projection, predicate meaning, + or parent complex offsets. +- **Complex plan contract:** The shape-owning leaf's definition/repetition levels are parsed once + into parent-row boundaries and child-presence/null decisions. Sibling physical streams consume + the same parent-row range and validate their counts so offsets and null maps cannot drift. + ## 3. From File Open to Scan Plan After a reader receives a Split, it opens the file and builds the scan plan. This phase determines @@ -75,7 +135,7 @@ sequenceDiagram FS->>TR: prepare/open split TR->>TR: Map schema and localize predicates TR->>PR: FileScanRequest - PR->>FC: Open FileReader + PR->>FC: Open FileReader / native stream adapter FC->>META: Read footer and schema META-->>PR: Row Group / Column Chunk metadata PR->>PLAN: Candidate Row Groups and local predicates @@ -94,8 +154,17 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, - FileCache prefetch, and MergeRange routing. +- **ParquetFileContext:** Adapts Doris FileReader to the native metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. Its immutable footer cache entry owns + the V2 `NativeFieldDescriptor` tree directly; repeated V2 opens neither re-read the footer nor + serialize and parse another metadata representation. + +For every selected Row Group, native dictionary/index probes finish on the cached metadata tree first. The +scheduler then computes projected physical Column Chunk ranges and installs one shared native +`MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and +lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is +disabled in this mode, avoiding duplicate buffers. Large chunks and in-memory files use the base +reader, while remote FileCache prefetch remains the non-MergeRange path. > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data @@ -279,6 +348,258 @@ flowchart LR > output columns must decode and copy. This is the main benefit of lazy materialization in a > columnar format. +### 7.1 Selection and Cursor Contract + +The native decoder receives a sorted selection over logical rows. Logical positions include nulls; +they are not offsets into the encoded non-null payload. Definition levels and selection are merged +as two ordered streams into four run types: + +| Run | Consume logical level entries | Consume encoded payload | Append output | +| --- | --- | --- | --- | +| Selected non-null (`CONTENT`) | Yes | Yes | Value | +| Selected null (`NULL_DATA`) | Yes | No | Default plus null-map bit | +| Filtered non-null (`FILTERED_CONTENT`) | Yes | Yes or decoder-native skip | No | +| Filtered null (`FILTERED_NULL`) | Yes | No | No | + +The contract keeps three counts explicit: logical level entries consumed, encoded non-null values +consumed, and Doris output values appended. The page reader may cross page boundaries while +satisfying one batch, but all three counts must be aligned when it returns. A dense identity +selection, an empty selection, and an arbitrary fragmented selection use the same decoder API. +Selection inputs are borrowed only for the duration of the decode call. + +For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes +definition-level runs and builds the four-way selection plan in persistent scratch whose capacity +survives adaptive batch changes. The plan is normalized to sorted physical non-NULL ranges and +enters `DataTypeSerDe` and the encoding decoder once. If selected NULLs are interleaved, the reader +separately records their logical positions, decodes the compact physical payload, and expands the +final scalar column backwards in place. This is the hybrid selection path: it keeps the dense +direct-materialization path, but moves sparse range traversal inside the concrete decoder instead +of repeatedly constructing a SerDe consumer for every selected or NULL run. + +The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk +copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding validates +IDs in bounded batches and either gathers repeated/literal runs directly or builds one compact +selected-ID batch when a sparse lookup would exceed the L2 cache; BOOLEAN, +DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected +values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate +Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding +state even when it is not copied, preventing the next batch from decoding shifted values. + +Scalar destinations with in-place expansion no longer use the per-run NULL fallback. The fallback +remains only for unsupported destination shapes and is visible through +`HybridSelectionNullFallbackBatches`; it is a correctness boundary, not an Arrow fallback. + +### 7.2 Page and Encoding Kernel + +The native page reader parses Page V1 and Page V2 headers, obtains level streams and payload bytes, +decompresses only the required region, and installs a decoder selected from physical type plus data +encoding. Page V1 and V2 differ in where levels are stored and which bytes are compressed; after +that parsing step both feed the same level and value-decoder contracts. + +| Encoding family | Native responsibility | +| --- | --- | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, validate IDs in bounded batches, fuse RLE runs with direct gather for cache-resident dictionaries, and use one compact selected-ID gather for large sparse dictionaries | +| RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | +| DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value state, then retain selected references only | +| BYTE_STREAM_SPLIT | Reassemble selected primitive lane ranges directly into one compact batch without an Arrow intermediate | + +Unsupported physical-type/encoding combinations return an explicit error. They never fall back to +Arrow and never produce a plausible result through a decoder selected only by logical Doris type. +Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. + +Read and skip are the same trust boundary. Every decoder validates the requested count against its +declared/remaining values before pointer advancement, allocation, multiplication, or integer +narrowing. BYTE_ARRAY dictionaries bound the entry count by the available four-byte length prefixes +before reserving storage and validate every decoded ID. DELTA_BYTE_ARRAY requires each prefix to fit +the preceding reconstructed value and checks the reconstructed and aggregate byte lengths. BOOLEAN +RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a +partially consumed cursor as if the request succeeded. + +Level bit width is only a storage bound: for a schema maximum of 2, two bits can still encode the +invalid value 3. Batch, run, and single-value level cursors therefore reject every decoded value +above the schema maximum. Nested page traversal reserves only the requested parent-row frontier; +an untrusted Page V1/V2 `num_values` cannot trigger an eager page-sized allocation before the level +payload proves those entries exist. + +Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the +destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce +exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and +uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead +of a buffer overrun, silent truncation, or plausible shifted values. + +Before a stream is created, scalar and levels-only readers call one signed, file-bounded Column +Chunk range validator. Writer compatibility derived from `created_by` may add at most 100 bytes of +file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page +iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page +ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are +strictly increasing, its first location equals the owning `data_page_offset`, and every location +remains inside that validated Column Chunk. The metadata anchor rejects uniformly shifted indexes +that row monotonicity alone cannot detect; otherwise sequential traversal preserves correctness. + +### 7.3 Direct Materialization and Scratch Reuse + +Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected +`DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width +types append contiguous runs. PLAIN BYTE_ARRAY publishes payload offsets, cumulative destination +offsets, and coalesced survivor spans so string columns perform larger range copies without a +`StringRef[]` staging array. Other binary encodings may use persistent references that remain valid +only while the page or dictionary buffer is pinned by the persistent leaf reader. + +The direct path materializes the projected file type. Table-schema changes, including numeric +widening, decimal precision/scale changes, and string/date conversions, are represented by +`ColumnMapper` and executed by `TableReader` after file-local predicates finish. The native reader +rejects a different requested type because accepting it would move schema evolution into physical +decode and could change predicate semantics. No intermediate physical-value batch is introduced. + +`DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded +physical values and is useful to generic format conversion code, but routing every Parquet value +through it would recreate an intermediate materialization layer. The v2 native path does not use it: +ColumnReader handles levels/selection, Decoder parses encoding streams, and DataTypeSerDe performs +physical/logical conversion while appending to the final column. + +Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian +two's-complement values with correct sign extension, and apply precision/scale conversion exactly +once. Decimal integer and binary inputs stay in a 256-bit intermediate through exact scale-down, +scale-up overflow, and target-precision checks; discarded non-zero digits are conversion failures, +and narrowing happens only after success. Date, timestamp, INT96, unsigned annotations, +CHAR/VARCHAR, and timezone conversions retain +the same semantic checks as the general conversion path. A fast path is enabled only when those +checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and +`TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct +local/unspecified semantics; data decode and statistics pruning share this interpretation. +Timestamp conversion validates millisecond scaling before multiplication, validates INT96 +nanos-of-day before widened Julian-day arithmetic, and rejects values outside Doris years +0001-9999. Plain, dictionary, and decoded-value inputs share these bounds. + +Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode +stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and +decimal conversion failures. Strict mode and non-nullable targets return the error. Typed +dictionary materialization records failing dictionary entries once and propagates those failures +through decoded dictionary IDs, so dictionary and plain pages have identical behavior. + +The typed dictionary is materialized through the logical SerDe once per decoder dictionary +generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving +row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation +change invalidates it, and every ID is checked before access. Cache-resident or dense reads stream +validated RLE runs directly into the destination column without a page-sized ID vector. Sparse +reads whose dictionary exceeds L2 retain only selected IDs, then perform one gather; malformed late +IDs roll back the destination to its pre-batch size. + +The persistent leaf reader owns reusable conversion objects, null map, selection ranges, +definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris +column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. +After the top-level complex reader has consumed the level plan, retained capacity above the 4 MiB +high-water limit becomes eligible for release. The reader accounts separately for active and +retained bytes, including decoder-owned buffers: active oversized scratch is never released, and +eligible capacity is released only after three consecutive ordinary/idle batches. This hysteresis +prevents a repeated-value outlier from being pinned for the Row Group without turning a legitimate +large steady-state batch into allocate/free thrash. The native path does not create an Arrow +builder or Arrow array. + +### 7.4 Complex Types and Parent Shape Plans + +Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One +parent row may contain zero, one, or many level entries, and its final entry can reside on the next +page. The complex reader therefore builds a parent shape plan from one owning leaf with: + +- parent-row start/end boundaries derived from repetition levels; +- ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from + definition thresholds; +- child payload positions and selected parent rows; +- cross-page continuation state for an unfinished parent row. + +ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores separate level +streams for separate physical leaves, so sibling readers still consume their own streams; they must +advance over the same parent-row range and validate their payload counts instead of redefining the +parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses +the key leaf as the entry-shape owner, requires the materialized key and value columns to match that +outer entry count, and validates Parquet's non-null key requirement rather than repairing it. Raw +key/value repetition vectors are intentionally not compared because a nested MAP value owns +additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent +boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. + +Level scratch grows with level entries that were actually decoded, while its initial reservation is +bounded by the requested parent-row frontier rather than the page header's untrusted value count. +Long repeated rows can still grow incrementally beyond that frontier, and long null/non-null runs +are split into representable internal runs without introducing a new row boundary. Tests cover null +ancestors, empty collections, null elements/values, nested +STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and +batches. + +#### Complex-reader interface and materialization cost + +The original v2 prototype placed an Arrow-decoded leaf batch and separate load/build/consume phases +between encoded pages and Doris columns. That design required implicit phase ordering, duplicated +level traversal in ARRAY/MAP/STRUCT wrappers, and retained decoded binary payload even when a caller +needed only nullability. The prototype hierarchy and its batch container have been removed. + +The production boundary is now the same compact cursor contract used for scalar columns: +`NativeColumnReader::read/select/skip`. Its persistent native reader owns page, decoder, level, +conversion, and complex-column state and materializes the complete result directly into the caller's +Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris +column, or public load-before-build protocol participates in predicate or output scans. + +Internally, complex decoding still has to solve the parent-shape problem. For example, levels +representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, +and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates +the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind +the native reader boundary rather than in caller-visible phases. + +When schema evolution makes every projected STRUCT child missing, the native reader retains one +physical reference leaf solely for shape. Its levels-only interface advances definition, +repetition, and encoded payload cursors (including dictionary-index validation), counts STRUCT +instances from the parent thresholds, and discards the payload without allocating a temporary +string or nested Doris column. + +`CountColumnReader` selects one representative leaf (the key for MAP) and uses the v2 native +`LevelReader` to consume definition/repetition levels without decoding payload values. It exposes +neither decoded values nor the ordinary scan-reader API, so COUNT pushdown cannot become a value +fallback and large complex payloads never enter aggregate state. + +Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and +the collection reader consumes persistent level buffers. DuckDB provides the same useful design +principle through its single `Read(input, vector)` boundary and reusable child vectors. See DuckDB's +official [LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +[STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), +and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). + +### 7.5 Index Coordinate Domains and Composition + +Index correctness depends on keeping its coordinate systems separate: + +| Identity | Scope | Must not be confused with | +| --- | --- | --- | +| Table/local column ID | Current file request and Doris block | Physical Parquet leaf ordinal | +| Physical leaf-column ID | Footer Row Group Column Chunk array | Logical parent STRUCT/ARRAY/MAP ordinal | +| Row Group ID | File metadata | Split ordinal or batch ordinal | +| Data-page ordinal | One Column Chunk, excluding dictionary pages | PageHeader sequence including dictionary page | +| OffsetIndex row ordinal | Logical row inside a Row Group | Encoded non-null value position | +| Selection index | Logical row inside the current batch, nulls included | Dictionary ID or compact output position | +| Dictionary-entry ID | One Column Chunk dictionary | Row ordinal, global dictionary ID, or the next Row Group's dictionary | + +Row Group statistics, dictionary pruning, Bloom, ColumnIndex/OffsetIndex, page skip plans, cache +ranges, row selection, and lazy predicate/output readers are composed in that order. Each stage may +only remove candidates already expressed in the same Row Group logical-row domain. Page ordinals +from ColumnIndex and OffsetIndex are validated together before they become `selected_ranges` and a +per-leaf physical page-skip plan. + +Dictionary row filtering starts only when metadata proves every data page in the Column Chunk uses +a dictionary encoding. The predicate is evaluated against the current dictionary to produce an +entry bitmap; decoded IDs are checked against both dictionary length and bitmap length. A mixed +dictionary/plain transition falls back before consuming data. Once selected dictionary reading has +advanced a page cursor, loss of dictionary output is corruption rather than a retry through another +path with shifted state. + +Missing optional indexes or unsupported predicate/type combinations retain rows. Malformed +offsets, inconsistent page counts, out-of-range dictionary IDs, overlapping/unsorted invalid ranges, +or an impossible cursor relationship are reported as corruption. Cache hits and misses do not +change any page, level, value, or dictionary cursor. + ## 8. Supported Indexes and Their Boundaries V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal @@ -315,12 +636,12 @@ flowchart TD ## 9. Cache and I/O Optimization -Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized -Parquet ranges, cache predicate results, and merge small random reads. +Parquet V2 has five complementary cache and I/O paths: cache footer/parsed metadata, cache remote +file blocks, cache serialized Parquet ranges, cache predicate results, and merge small random reads. ```mermaid flowchart TB - A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + A[Parquet Column Reader Range Read] --> B{Parquet Page Cache Hit?} B -- "Yes" --> C[Return Cached Serialized Range Bytes] B -- "No" --> D{MergeRange Active?} D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] @@ -334,11 +655,46 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | +| Footer metadata cache | V2-owned immutable Thrift metadata and native physical schema tree | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, and unsafe cross-version cache casts | +| Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | | MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | +### Footer Cache Parity with V1 + +V2 reuses the common stable file identity policy, but owns its footer parser, schema lifecycle, +metadata payload type, and cache-key suffix entirely under `format_v2`. A v2 cache entry can be +shared by v2 readers but can never be cast as a v1 metadata object. Mutable Row Group, selection, +decoder, and scratch state remains per reader. Path-only keys are insufficient. Parse failures, +short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a +successful entry. No change to the v1 parser or metadata cache behavior is required. + +On a v2 cold miss, the parser deserializes the footer once into the V2-owned Thrift object and builds +the native schema tree directly from `SchemaElement` into V2's `NativeFieldDescriptor`. Row-group statistics, +dictionary and Bloom metadata planning consume the same native tree; no Arrow `FileMetaData` tree +or serialized-footer copy is retained. Production ColumnIndex/OffsetIndex planning reads and parses +Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and validated +OffsetIndex objects are transferred into Row Group execution instead of being fetched a second +time. Arrow metadata and PageIndexReader paths remain only as test oracles. + +Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper +loads from byte zero on its first physical access, whether that access is a cold footer read or a +warm data-page read after a footer-cache hit. This preserves identical cold/warm behavior for HTTP +servers that advertise Range support but answer an overlong near-EOF range with HTTP 200, and keeps +the compatibility policy out of the v1 reader. + +### Page Cache Parity with V1 + +The native path uses v1's page-cache semantics as its minimum contract: the same stable file +identity, Column Chunk/page byte ranges, compressed versus decompressed entry distinction, checksum +and decompression ownership, subrange coverage, invalidation, admission, and fallback I/O rules. +Mutable decoder and dictionary state is never cached. A hit returns immutable bytes and advances the +reader exactly as the corresponding base read would; a miss performs normal I/O before optional +admission. An alternative policy is accepted only with correctness coverage plus warm/cold, +selective/dense, local/remote benchmarks showing it is no worse than v1. + ### Why Page Cache registers only surviving chunks The footer is read before Row Group planning and before Page Cache ranges are registered, so @@ -349,8 +705,10 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. +- After dictionary probing, small projected chunks share one Row-Group-scoped + MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base + reader. Native metadata/index reads use explicit immutable ranges and never own a decoder stream + cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -396,6 +754,14 @@ flowchart LR E --> F[Bound by batch_size and Selected Range] ``` +Changing the requested row cap changes only the amount of work performed by the persistent reader. +It does not recreate Column Readers, Column Chunk readers, dictionaries, SerDe/conversion objects, +builders, or scratch. The probe is charged as a normal batch and estimates completed Doris rows and +bytes after table-level materialization, matching v1's measurement point. Empty or highly filtered +probes do not permanently collapse the batch size; the scheduler retains enough evidence before +updating its estimate and respects page/range boundaries without turning each fragment into a new +reader lifecycle. + ### 10.3 Aggregate Pushdown When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX @@ -408,6 +774,34 @@ Without row-level filtering, output columns may be warmed together. With filteri columns first and defer non-predicate columns until at least one row survives, aligning network bandwidth with lazy materialization. +For safe single-column conjuncts, the scheduler records an exponentially weighted cost per input +row and survival ratio. Cold batches preserve declaration order; after every candidate has a sample, +the next batch orders predicates by cost per rejected row. The same learned order limits predicate +prefetch to the prefix with a meaningful probability of being reached, and a sustained high overall +survival ratio estimate may warm output chunks early. This retains correctness because only +independently safe conjuncts move and all readers still advance over the same logical batch. + +Predicate-only `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons can bypass Doris-column +materialization when every value page in the Column Chunk is fixed-width PLAIN and the expression +is an exact `=`, `!=`, `<`, `<=`, `>`, or `>=` comparison with a literal. The scheduler compiles +the conjunction into physical-value predicates, the decoder compares selected non-NULL spans in +place, and the reader remaps the compact match bytes to logical nullable rows. NULL comparisons are +false. Mixed encodings, projected predicate columns, casts, logical conversions, compound +expressions, and unsupported types make the decision before either the definition-level or value +cursor advances, so they safely retain ordinary expression evaluation. An exact direct result +leaves only a row-shaped placeholder for the hidden predicate slot and avoids both predicate-column +materialization and later predicate compaction. + +Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by +selection generation and batch shape, so every lazily materialized output reader consumes the same +bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary +capacity remains reusable. + +Fully rejected batches accumulate a logical lag for lazy columns. When a later batch survives, each +lazy reader consumes that lag in at most 65,535-row dense-filter chunks. This keeps the scheduler's +single logical skip while bounding adapter-owned bitmap capacity independently of prefix length and +lazy projection width. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -442,6 +836,15 @@ split safely, or read anomalies must never change query semantics. Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and then I/O/cache health. Total ScanTime alone does not identify the cause. +The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the +format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, +table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are +charged to their owning layer. Native child-reader statistics accumulate in plain integers and are +published every 16 batches. Row Group reset, EOF, and reader close force the final delta before +destroying the reader tree, so short files retain complete attribution without paying recursive +profile publication on every tiny batch. Retained-scratch inspection uses the same amortized cadence +and a Row Group remains its hard lifetime bound. + ```mermaid flowchart TD A[Slow Scan] --> B{Many Row Groups Pruned?} @@ -465,10 +868,18 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | +| PLAIN direct predicate | How many eligible predicate-only physical batches and input rows bypassed Doris-column materialization? Inspect `PlainPredicateDirectBatches/Rows`. | +| Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | +| Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? Compare `HybridSelectionBatches`, `HybridSelectionRanges`, and `HybridSelectionNullFallbackBatches` to distinguish batched sparse decode from NULL-interleaving fallback. | +| Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | +| Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | +| Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | > Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max > ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work @@ -476,14 +887,16 @@ flowchart TD ## 13. Summary -The FileScannerV2 Parquet scan pipeline has three primary threads: +The FileScannerV2 Parquet scan pipeline has four primary threads: 1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local semantics, preserving schema evolution, partition columns, and missing columns. 2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, Statistics, Dictionary, Bloom, Page Index, and actual-value filters. -3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, - FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. +3. **Decode thread:** Persistent native page/encoding readers merge Dremel levels with Selection, + reuse scratch, reconstruct complex values from shared plans, and materialize directly to Doris. +4. **I/O thread:** Predicate-first reads, adaptive batches, Footer/File/Page/Condition caches, and + MergeRange reduce read amplification together. ```mermaid flowchart LR diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index f59d9a03da9f1a..29848358d63ca3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -1236,10 +1236,6 @@ public long getCloudMetaTimeMs() { return TimeUnit.NANOSECONDS.toMillis(getPartitionVersionTime + getTableVersionTime); } - public void addExternalCatalogMetaTime(long ms) { - this.externalCatalogMetaTime += ms; - } - public long getExternalCatalogMetaTimeMs() { return externalCatalogMetaTime; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 8df635e5ea5f96..bfe34e9a32325e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -47,6 +47,7 @@ import org.apache.doris.statistics.StatisticalType; import org.apache.doris.system.Backend; import org.apache.doris.tablefunction.ExternalFileTableValuedFunction; +import org.apache.doris.thrift.TColumnCategory; import org.apache.doris.thrift.TExternalScanRange; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; @@ -185,11 +186,9 @@ protected void initSchemaParams() throws UserException { for (SlotDescriptor slot : desc.getSlots()) { TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); slotInfo.setSlotId(slot.getId().asInt()); - boolean isFileSlot = !partitionKeys.contains(slot.getColumn().getName()); - if (isIcebergRowIdColumn(slot)) { - isFileSlot = false; - } - slotInfo.setIsFileSlot(isFileSlot); + TColumnCategory category = classifyColumn(slot, partitionKeys); + slotInfo.setCategory(category); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false); @@ -202,23 +201,42 @@ protected void initSchemaParams() throws UserException { } private void updateRequiredSlots() throws UserException { + Map existingSlotInfoById = Maps.newHashMap(); + if (params.getRequiredSlots() != null) { + for (TFileScanSlotInfo slotInfo : params.getRequiredSlots()) { + existingSlotInfoById.put(slotInfo.getSlotId(), slotInfo); + } + } params.unsetRequiredSlots(); + List partitionKeys = getPathPartitionKeys(); for (SlotDescriptor slot : desc.getSlots()) { - TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); - slotInfo.setSlotId(slot.getId().asInt()); - boolean isFileSlot = !getPathPartitionKeys().contains(slot.getColumn().getName()); - if (isIcebergRowIdColumn(slot)) { - isFileSlot = false; + TFileScanSlotInfo slotInfo = existingSlotInfoById.get(slot.getId().asInt()); + if (slotInfo == null) { + slotInfo = new TFileScanSlotInfo(); } - slotInfo.setIsFileSlot(isFileSlot); + slotInfo.setSlotId(slot.getId().asInt()); + TColumnCategory category = classifyColumn(slot, partitionKeys); + slotInfo.setCategory(category); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } // Update required slots and column_idxs in scanRangeLocations. setColumnPositionMapping(); } - private boolean isIcebergRowIdColumn(SlotDescriptor slot) { - return Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName()); + /** + * Classify a column's category for the BE reader. + * Subclasses override this for format-specific classification. + */ + protected TColumnCategory classifyColumn(SlotDescriptor slot, List partitionKeys) { + if (partitionKeys.contains(slot.getColumn().getName())) { + return TColumnCategory.PARTITION_KEY; + } + return TColumnCategory.REGULAR; + } + + protected boolean isFileSlot(TColumnCategory category) { + return category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED; } public void setTableSample(TableSample tSample) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index a17ff87b1131c5..0c78be1accf76c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,8 +40,10 @@ public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnap this.snapshot = snapshot; this.nameMapping = nameMapping.map(mapping -> { Map> copy = new HashMap<>(); - mapping.forEach((id, names) -> copy.put(id, List.copyOf(names))); - return Map.copyOf(copy); + // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. + mapping.forEach((id, names) -> copy.put(id, + Collections.unmodifiableList(new ArrayList<>(names)))); + return Collections.unmodifiableMap(copy); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d989ed0cf32ab6..d4167d7bc8aa40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -18,9 +18,11 @@ package org.apache.doris.datasource.iceberg.source; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 13d895ad7e4e84..edfed3c249ec4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -883,7 +883,6 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca .map(exprId -> Objects.requireNonNull(context.findSlotRef(exprId), "missing slot for pushed-down COUNT argument " + exprId).getSlotId()) .collect(Collectors.toList())); - scanNode.setHasPartitionPredicate(fileScan.hasPartitionPredicate()); TableNameInfo tableNameInfo = new TableNameInfo(null, "", ""); TableRefInfo ref = new TableRefInfo(tableNameInfo, null, null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index cd0faffeb5edbd..462806e3294759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -570,6 +570,7 @@ private LogicalAggregate storageLayerAggregate( Map, PushDownAggOp> supportedAgg = PushDownAggOp.supportedFunctions(); boolean containsCount = false; + boolean containsCountStar = false; boolean countHasCastArgument = false; Set checkNullSlots = new HashSet<>(); Set expressionAfterProject = new HashSet<>(); @@ -586,7 +587,9 @@ private LogicalAggregate storageLayerAggregate( // Check if contains Count function if (functionClass.equals(Count.class)) { containsCount = true; - if (!function.getArguments().isEmpty()) { + if (function.getArguments().isEmpty()) { + containsCountStar = true; + } else { Expression arg0 = function.getArguments().get(0); if (arg0 instanceof SlotReference) { checkNullSlots.add((SlotReference) arg0); @@ -734,10 +737,21 @@ private LogicalAggregate storageLayerAggregate( } } if (mergeOp == PushDownAggOp.COUNT || mergeOp == PushDownAggOp.MIX) { - // NULL value behavior in `count` function is zero, so - // we should not use row_count to speed up query. the col - // must be not null - if (column.isAllowNull() && checkNullSlots.contains(slot)) { + if (logicalScan instanceof LogicalFileScan && mergeOp == PushDownAggOp.COUNT + && containsCountStar && column.isAllowNull() && checkNullSlots.contains(slot)) { + // One metadata cardinality cannot represent both COUNT(*) and the smaller + // COUNT(nullable_col); synthetic rows would make one upper aggregate wrong. + return canNotPush; + } + // Nullable file COUNT is exact only when this query is routed to FileScannerV2, + // which carries the semantic argument and counts definition levels. Gating on the + // session switch keeps V1 on its original full-column evaluation path. + boolean supportsNullableFileCount = logicalScan instanceof LogicalFileScan + && mergeOp == PushDownAggOp.COUNT + && cascadesContext.getConnectContext() != null + && cascadesContext.getConnectContext().getSessionVariable().enableFileScannerV2; + if (column.isAllowNull() && checkNullSlots.contains(slot) + && !supportsNullableFileCount) { return canNotPush; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 1d4255b6df98ef..ac2d77a4fce083 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; -import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -97,30 +96,30 @@ public String toString() { } public PhysicalStorageLayerAggregate withPhysicalOlapScan(PhysicalOlapScan physicalOlapScan) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( - physicalOlapScan, aggOp, countArgumentExprIds)); + // branch-4.1 predates copyWithSameId; its plan-copy contract creates a fresh node ID. + return new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp, countArgumentExprIds); } @Override public PhysicalStorageLayerAggregate withGroupExpression(Optional groupExpression) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, - countArgumentExprIds, groupExpression, getLogicalProperties(), physicalProperties, statistics)); + return new PhysicalStorageLayerAggregate(relation, aggOp, countArgumentExprIds, + groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, - countArgumentExprIds, groupExpression, logicalProperties.get(), physicalProperties, statistics)); + return new PhysicalStorageLayerAggregate(relation, aggOp, countArgumentExprIds, + groupExpression, logicalProperties.get(), physicalProperties, statistics); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( + return new PhysicalStorageLayerAggregate( (PhysicalCatalogRelation) relation.withPhysicalPropertiesAndStats(null, statistics), aggOp, countArgumentExprIds, groupExpression, - getLogicalProperties(), physicalProperties, statistics)); + getLogicalProperties(), physicalProperties, statistics); } /** PushAggOp */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index ccf6bf3fd79de9..bd03966d5576a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -59,6 +59,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index aee814907bce61..21a899ac673f00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -17,30 +17,64 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TColumnCategory; +import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileScanSlotInfo; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; public class FileQueryScanNodeTest { private static final long MB = 1024L * 1024L; + private static final Method UPDATE_REQUIRED_SLOTS_METHOD; + + static { + try { + UPDATE_REQUIRED_SLOTS_METHOD = FileQueryScanNode.class.getDeclaredMethod("updateRequiredSlots"); + UPDATE_REQUIRED_SLOTS_METHOD.setAccessible(true); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private TableIf table; private static class TestFileQueryScanNode extends FileQueryScanNode { + private TableIf targetTable; + TestFileQueryScanNode(SessionVariable sv) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), "test", ScanContext.EMPTY, false, sv); } + TupleDescriptor getTupleDescriptor() { + return desc; + } + + void setTargetTable(TableIf targetTable) { + this.targetTable = targetTable; + } + @Override protected TFileFormatType getFileFormatType() throws UserException { return TFileFormatType.FORMAT_ORC; @@ -53,7 +87,7 @@ protected List getPathPartitionKeys() throws UserException { @Override protected TableIf getTargetTable() throws UserException { - return null; + return targetTable; } @Override @@ -62,6 +96,12 @@ protected Map getLocationProperties() throws UserException { } } + @Before + public void setUp() { + table = Mockito.mock(TableIf.class); + Mockito.when(table.getName()).thenReturn("test_table"); + } + @Test public void testApplyMaxFileSplitNumLimitRaisesTargetSize() { SessionVariable sv = new SessionVariable(); @@ -88,4 +128,38 @@ public void testApplyMaxFileSplitNumLimitDisabled() { long target = node.applyMaxFileSplitNumLimit(32 * MB, 10_000L * MB); Assert.assertEquals(32 * MB, target); } + + @Test + public void testUpdateRequiredSlotsPreservesInlineDefaultValueExpr() throws Exception { + SessionVariable sv = new SessionVariable(); + TestFileQueryScanNode node = new TestFileQueryScanNode(sv); + node.setTargetTable(table); + + TupleDescriptor desc = node.getTupleDescriptor(); + desc.setTable(table); + SlotDescriptor slot = new SlotDescriptor(new SlotId(1), desc); + slot.setColumn(new Column("c1", Type.INT)); + desc.addSlot(slot); + Mockito.when(table.getFullSchema()).thenReturn(Arrays.asList(slot.getColumn())); + + TExpr defaultExpr = new TExpr(); + defaultExpr.setNodes(Collections.emptyList()); + TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); + slotInfo.setSlotId(slot.getId().asInt()); + slotInfo.setCategory(TColumnCategory.REGULAR); + slotInfo.setIsFileSlot(true); + slotInfo.setDefaultValueExpr(defaultExpr); + + TFileScanRangeParams params = new TFileScanRangeParams(); + params.setRequiredSlots(Arrays.asList(slotInfo)); + node.params = params; + + UPDATE_REQUIRED_SLOTS_METHOD.invoke(node); + + TFileScanSlotInfo updatedSlotInfo = node.params.getRequiredSlots().get(0); + Assert.assertSame(slotInfo, updatedSlotInfo); + Assert.assertTrue(updatedSlotInfo.isSetDefaultValueExpr()); + Assert.assertSame(defaultExpr, updatedSlotInfo.getDefaultValueExpr()); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java index 39fc15ddbe1a7e..3c6aa2676803de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java @@ -21,6 +21,8 @@ import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; @@ -45,7 +47,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * Test for IcebergWriterHelper DeleteFile conversion @@ -79,16 +80,16 @@ public void testConvertToWriterResultRespectsNoneMetricsMode() { Mockito.when(table.schema()).thenReturn(schema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setColumnSizes(Map.of(2, 128L)); - columnStats.setValueCounts(Map.of(2, 10L)); - columnStats.setNullValueCounts(Map.of(2, 0L)); - columnStats.setLowerBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x01}))); - columnStats.setUpperBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x02}))); + columnStats.setColumnSizes(ImmutableMap.of(2, 128L)); + columnStats.setValueCounts(ImmutableMap.of(2, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L)); + columnStats.setLowerBounds(ImmutableMap.of(2, ByteBuffer.wrap(new byte[] {0x01}))); + columnStats.setUpperBounds(ImmutableMap.of(2, ByteBuffer.wrap(new byte[] {0x02}))); TIcebergCommitData commitData = new TIcebergCommitData(); commitData.setFilePath("/path/to/data.parquet"); @@ -96,7 +97,7 @@ public void testConvertToWriterResultRespectsNoneMetricsMode() { commitData.setFileSize(1024); commitData.setColumnStats(columnStats); - WriteResult result = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)); + WriteResult result = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)); DataFile dataFile = result.dataFiles()[0]; Assertions.assertTrue(dataFile.columnSizes() == null || dataFile.columnSizes().isEmpty()); @@ -112,17 +113,17 @@ public void testConvertToWriterResultCountsModeOmitsBounds() { Mockito.when(table.schema()).thenReturn(schema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setColumnSizes(Map.of(2, 128L)); - columnStats.setValueCounts(Map.of(2, 10L)); - columnStats.setNullValueCounts(Map.of(2, 0L)); - columnStats.setLowerBounds(Map.of( + columnStats.setColumnSizes(ImmutableMap.of(2, 128L)); + columnStats.setValueCounts(ImmutableMap.of(2, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L)); + columnStats.setLowerBounds(ImmutableMap.of( 2, Conversions.toByteBuffer(Types.StringType.get(), "abcdefgh"))); - columnStats.setUpperBounds(Map.of( + columnStats.setUpperBounds(ImmutableMap.of( 2, Conversions.toByteBuffer(Types.StringType.get(), "ijklmnop"))); TIcebergCommitData commitData = new TIcebergCommitData(); @@ -131,11 +132,11 @@ public void testConvertToWriterResultCountsModeOmitsBounds() { commitData.setFileSize(1024); commitData.setColumnStats(columnStats); - DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; - Assertions.assertEquals(Map.of(2, 128L), dataFile.columnSizes()); - Assertions.assertEquals(Map.of(2, 10L), dataFile.valueCounts()); - Assertions.assertEquals(Map.of(2, 0L), dataFile.nullValueCounts()); + Assertions.assertEquals(ImmutableMap.of(2, 128L), dataFile.columnSizes()); + Assertions.assertEquals(ImmutableMap.of(2, 10L), dataFile.valueCounts()); + Assertions.assertEquals(ImmutableMap.of(2, 0L), dataFile.nullValueCounts()); Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); } @@ -149,15 +150,15 @@ public void testConvertToWriterResultTruncatesStringAndBinaryBounds() { Mockito.when(table.schema()).thenReturn(boundsSchema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(3)")); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setLowerBounds(Map.of( + columnStats.setLowerBounds(ImmutableMap.of( 1, Conversions.toByteBuffer(Types.StringType.get(), "abcdef"), 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); - columnStats.setUpperBounds(Map.of( + columnStats.setUpperBounds(ImmutableMap.of( 1, Conversions.toByteBuffer(Types.StringType.get(), "uvwxyz"), 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); @@ -167,7 +168,7 @@ public void testConvertToWriterResultTruncatesStringAndBinaryBounds() { commitData.setFileSize(1024); commitData.setColumnStats(columnStats); - DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; Assertions.assertEquals("abc", Conversions.fromByteBuffer( Types.StringType.get(), dataFile.lowerBounds().get(1)).toString()); @@ -185,13 +186,13 @@ public void testConvertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSucce Mockito.when(table.schema()).thenReturn(boundsSchema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "orc", TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(1)")); String maxWithoutSuccessor = new String(Character.toChars(Character.MAX_CODE_POINT)) + "tail"; TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setUpperBounds(Map.of( + columnStats.setUpperBounds(ImmutableMap.of( 1, Conversions.toByteBuffer(Types.StringType.get(), maxWithoutSuccessor))); TIcebergCommitData commitData = new TIcebergCommitData(); @@ -200,7 +201,7 @@ public void testConvertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSucce commitData.setFileSize(128); commitData.setColumnStats(columnStats); - DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; Assertions.assertNotNull(dataFile.upperBounds()); Assertions.assertEquals(maxWithoutSuccessor, Conversions.fromByteBuffer( @@ -213,7 +214,7 @@ public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { Mockito.when(table.schema()).thenReturn(schema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); @@ -227,7 +228,7 @@ public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { secondCommit.setRowCount(20); secondCommit.setFileSize(2048); - IcebergWriterHelper.convertToWriterResult(table, List.of(firstCommit, secondCommit)); + IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(firstCommit, secondCommit)); // One schema lookup is made by Iceberg's policy builder and one is captured for all files in the batch. Mockito.verify(table, Mockito.times(2)).schema(); @@ -236,7 +237,7 @@ public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { @Test public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@TempDir Path tempDir) { HadoopTables tables = new HadoopTables(new Configuration()); - Table baseTable = tables.create(schema, unpartitionedSpec, SortOrder.unsorted(), Map.of( + Table baseTable = tables.create(schema, unpartitionedSpec, SortOrder.unsorted(), ImmutableMap.of( TableProperties.FORMAT_VERSION, "3", TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(16)"), @@ -249,8 +250,8 @@ public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@Te ByteBuffer sequenceNumberBound = Conversions.toByteBuffer( MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), 3L); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setLowerBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); - columnStats.setUpperBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + columnStats.setLowerBounds(ImmutableMap.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + columnStats.setUpperBounds(ImmutableMap.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); TIcebergCommitData commitData = new TIcebergCommitData(); commitData.setFilePath("/path/to/v3-data.parquet"); @@ -260,7 +261,7 @@ public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@Te DataFile dataFile = Assertions.assertDoesNotThrow( () -> IcebergWriterHelper.convertToWriterResult( - transactionTable, List.of(commitData)).dataFiles()[0]); + transactionTable, ImmutableList.of(commitData)).dataFiles()[0]); Assertions.assertEquals(rowIdBound, dataFile.lowerBounds().get(rowId)); Assertions.assertEquals(rowIdBound, dataFile.upperBounds().get(rowId)); Assertions.assertEquals(sequenceNumberBound, dataFile.lowerBounds().get(sequenceNumberId)); @@ -279,15 +280,15 @@ public void testConvertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields Mockito.when(table.schema()).thenReturn(repeatedSchema); Mockito.when(table.spec()).thenReturn(unpartitionedSpec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "full")); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setColumnSizes(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L)); - columnStats.setValueCounts(Map.of(2, 2L, 4, 4L, 5, 5L, 6, 6L)); - columnStats.setNullValueCounts(Map.of(2, 0L, 4, 0L, 5, 0L, 6, 0L)); - columnStats.setLowerBounds(Map.of( + columnStats.setColumnSizes(ImmutableMap.of(2, 20L, 4, 40L, 5, 50L, 6, 60L)); + columnStats.setValueCounts(ImmutableMap.of(2, 2L, 4, 4L, 5, 5L, 6, 6L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L, 4, 0L, 5, 0L, 6, 0L)); + columnStats.setLowerBounds(ImmutableMap.of( 2, Conversions.toByteBuffer(Types.IntegerType.get(), 2), 4, Conversions.toByteBuffer(Types.StringType.get(), "key"), 5, Conversions.toByteBuffer(Types.StringType.get(), "value"), @@ -300,13 +301,13 @@ public void testConvertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields commitData.setFileSize(1024); commitData.setColumnStats(columnStats); - DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; - Assertions.assertEquals(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L), dataFile.columnSizes()); - Assertions.assertEquals(Map.of(6, 6L), dataFile.valueCounts()); - Assertions.assertEquals(Map.of(6, 0L), dataFile.nullValueCounts()); - Assertions.assertEquals(Map.of(6, columnStats.getLowerBounds().get(6)), dataFile.lowerBounds()); - Assertions.assertEquals(Map.of(6, columnStats.getUpperBounds().get(6)), dataFile.upperBounds()); + Assertions.assertEquals(ImmutableMap.of(2, 20L, 4, 40L, 5, 50L, 6, 60L), dataFile.columnSizes()); + Assertions.assertEquals(ImmutableMap.of(6, 6L), dataFile.valueCounts()); + Assertions.assertEquals(ImmutableMap.of(6, 0L), dataFile.nullValueCounts()); + Assertions.assertEquals(ImmutableMap.of(6, columnStats.getLowerBounds().get(6)), dataFile.lowerBounds()); + Assertions.assertEquals(ImmutableMap.of(6, columnStats.getUpperBounds().get(6)), dataFile.upperBounds()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 2742ac2aa6cd6f..731f7f860fdbc8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -17,12 +17,13 @@ package org.apache.doris.datasource.iceberg.source; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; -import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.CatalogIf; @@ -46,6 +47,8 @@ import org.apache.doris.thrift.TIcebergDeleteFileDesc; import org.apache.doris.thrift.TPushAggOp; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; @@ -123,10 +126,14 @@ public List getPathPartitionKeys() { return Collections.emptyList(); } - void addSlot(int slotId, Column column) { - SlotDescriptor slot = new SlotDescriptor(new SlotId(slotId), desc.getId()); - slot.setColumn(column); - desc.addSlot(slot); + @Override + public TableSnapshot getQueryTableSnapshot() { + return tableSnapshot; + } + + @Override + public TableScanParams getScanParams() { + return scanParams; } int enableAndGetIcebergScanSemanticsVersion() { @@ -220,53 +227,6 @@ public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefres } } - @Test - public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { - Schema systemTableSchema = new Schema( - Types.NestedField.required(1, "file_path", Types.StringType.get()), - Types.NestedField.required(2, "record_count", Types.LongType.get()), - Types.NestedField.optional(3, "readable_metrics", Types.StructType.of( - Types.NestedField.optional(4, "id", Types.StructType.of( - Types.NestedField.optional(5, "lower_bound", Types.IntegerType.get())))))); - Table systemTable = Mockito.mock(Table.class); - Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); - - TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); - setIcebergTable(node, systemTable); - node.addSlot(1, new Column("RECORD_COUNT", Type.BIGINT)); - node.addSlot(2, new Column(Column.GLOBAL_ROWID_COL + "system_table", Type.BIGINT)); - node.addSlot(3, new Column("FILE_PATH", Type.STRING)); - - List filters = Collections.singletonList(Expressions.greaterThan("record_count", 0L)); - Schema projectedSchema = node.getSystemTableProjectedSchema(filters, false); - - Assert.assertEquals(2, projectedSchema.columns().size()); - Assert.assertEquals("record_count", projectedSchema.columns().get(0).name()); - Assert.assertEquals("file_path", projectedSchema.columns().get(1).name()); - Assert.assertNull(projectedSchema.findField("readable_metrics")); - } - - @Test - public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws Exception { - Schema systemTableSchema = new Schema( - Types.NestedField.required(1, "file_path", Types.StringType.get()), - Types.NestedField.required(2, "record_count", Types.LongType.get())); - Table systemTable = Mockito.mock(Table.class); - Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); - - TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); - setIcebergTable(node, systemTable); - node.addSlot(1, new Column("record_count", Type.BIGINT)); - - try { - node.getSystemTableProjectedSchema( - Collections.singletonList(Expressions.equal("file_path", "data.parquet")), true); - Assert.fail("Filter columns must be materialized by the planner"); - } catch (UserException e) { - Assert.assertTrue(e.getMessage().contains("filter column file_path is not materialized")); - } - } - private static class CountPlanningIcebergScanNode extends IcebergScanNode { private final TableScan tableScan; private final long snapshotCount; @@ -352,16 +312,16 @@ public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws @Test public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { - Schema pinnedSchema = new Schema(11, List.of(Types.NestedField.optional("binary_default") + Schema pinnedSchema = new Schema(11, ImmutableList.of(Types.NestedField.optional("binary_default") .withId(7) .ofType(Types.BinaryType.get()) .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) .build())); Schema refreshedSchema = new Schema(12, - List.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + ImmutableList.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); Table refreshedTable = Mockito.mock(Table.class); Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); - Mockito.when(refreshedTable.schemas()).thenReturn(Map.of( + Mockito.when(refreshedTable.schemas()).thenReturn(ImmutableMap.of( pinnedSchema.schemaId(), pinnedSchema, refreshedSchema.schemaId(), refreshedSchema)); @@ -430,12 +390,12 @@ public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() t @Test public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { - Schema dataSnapshotSchema = new Schema(11, List.of(Types.NestedField.optional("string_default") + Schema dataSnapshotSchema = new Schema(11, ImmutableList.of(Types.NestedField.optional("string_default") .withId(7) .ofType(Types.StringType.get()) .withInitialDefault("not-base64") .build())); - Schema branchSchema = new Schema(12, List.of(Types.NestedField.optional("binary_default") + Schema branchSchema = new Schema(12, ImmutableList.of(Types.NestedField.optional("binary_default") .withId(7) .ofType(Types.BinaryType.get()) .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) @@ -443,7 +403,7 @@ public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws E Snapshot dataSnapshot = Mockito.mock(Snapshot.class); Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); Table table = Mockito.mock(Table.class); - Mockito.when(table.schemas()).thenReturn(Map.of( + Mockito.when(table.schemas()).thenReturn(ImmutableMap.of( dataSnapshotSchema.schemaId(), dataSnapshotSchema, branchSchema.schemaId(), branchSchema)); TableScan branchScan = Mockito.mock(TableScan.class); @@ -579,25 +539,6 @@ public void testSetIcebergParamsKeepsDeletionVectorOffsetAsLong() throws Excepti Assert.assertEquals((long) Integer.MAX_VALUE + 7L, deleteFileDesc.getContentSizeInBytes()); } - @Test - public void testSetIcebergParamsUsesSplitFileFormat() throws Exception { - TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); - String dataPath = "file:///tmp/data-file.orc"; - IcebergSplit split = new IcebergSplit(LocationPath.of(dataPath), 0, 128, 128, new String[0], - 2, Collections.emptyMap(), new ArrayList<>(), dataPath); - split.setTableFormatType(TableFormatType.ICEBERG); - split.setSplitFileFormat(FileFormat.ORC); - - Method method = IcebergScanNode.class.getDeclaredMethod("setIcebergParams", - TFileRangeDesc.class, IcebergSplit.class); - method.setAccessible(true); - - TFileRangeDesc rangeDesc = new TFileRangeDesc(); - method.invoke(node, rangeDesc, split); - - Assert.assertEquals(TFileFormatType.FORMAT_ORC, rangeDesc.getFormatType()); - } - @Test public void testPositionDeleteSystemTableValidatesDeletionVectorMetadata() throws Exception { DeleteFile deleteFile = Mockito.mock(DeleteFile.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index c15132cdc99deb..9dcf91731bbee2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -70,10 +70,18 @@ public class PaimonScanNodeTest { @Mock private PaimonFileExternalCatalog paimonFileExternalCatalog; + private PaimonSource mockPaimonSourceWithPartitionKeys(List partitionKeys) { + PaimonSource source = Mockito.mock(PaimonSource.class); + Table paimonTable = Mockito.mock(Table.class); + Mockito.when(paimonTable.partitionKeys()).thenReturn(partitionKeys); + Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); + return source; + } + @Test public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException { PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); - node.setSource(mockPaimonSourceWithPartitionKeys(Collections.emptyList())); + node.setSource(mockPaimonSourceWithPartitionKeys(Collections.emptyList())); List dataSplits = Arrays.asList( mockCountDataSplit("f1.parquet", 4_000), mockCountDataSplit("f2.parquet", 5_000), diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index 8ea9041480dadd..7645f0675074ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -32,6 +32,7 @@ import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TPushAggOp; +import com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -53,7 +54,7 @@ public void testCountColumnKeepsNormalFileSplitting() throws Exception { ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); Mockito.when(table.getTvf()).thenReturn(tvf); Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_LOCAL); - Mockito.when(tvf.getFileStatuses()).thenReturn(List.of( + Mockito.when(tvf.getFileStatuses()).thenReturn(ImmutableList.of( splittableFile("file:///tmp/count_col_1.parquet", 128 * MB), splittableFile("file:///tmp/count_col_2.parquet", 128 * MB))); desc.setTable(table); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index 1e7d6b39154b11..b49f8c8a19e84d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -30,6 +30,7 @@ import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -82,40 +83,6 @@ public void test2() throws Exception { Assertions.assertEquals("k2", slots.get(0).getColumn().getName()); } - @Test - public void testNestedColumnAccessPathInLazyMaterialize() throws Exception { - this.createTables("create table lazy_materialize_struct_tbl(" - + "id bigint, " - + "user_profile struct<" - + "personal:struct," - + "professional:struct>>) " - + "duplicate key(id) distributed by hash(id) buckets 1 " - + "properties('replication_num' = '1')"); - String sql = "select struct_element(struct_element(user_profile, 'professional'), 'skills') " - + "from lazy_materialize_struct_tbl order by id limit 10"; - - PlanChecker checker = PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .implement(); - PhysicalPlan plan = checker.getPhysicalPlan(); - plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); - PlanTranslatorContext translatorContext = new PlanTranslatorContext(checker.getCascadesContext()); - PlanFragment fragment = new PhysicalPlanTranslator(translatorContext).translatePlan(plan); - - List materializationNodes = Lists.newArrayList(); - fragment.getPlanRoot().collect(MaterializationNode.class, materializationNodes); - Assertions.assertEquals(1, materializationNodes.size()); - TupleDescriptor materializeTupleDesc = translatorContext.getTupleDesc( - materializationNodes.get(0).getTupleIds().get(0)); - SlotDescriptor userProfileSlot = materializeTupleDesc.getSlots().stream() - .filter(slot -> "user_profile".equals(slot.getColumn().getName())) - .findFirst() - .orElseThrow(() -> new AssertionError("lazy user_profile slot not found")); - Assertions.assertTrue(userProfileSlot.getAllAccessPaths().contains( - ColumnAccessPath.data(ImmutableList.of("user_profile", "professional", "skills")))); - } - @Test public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Exception { this.createTable("create table lazy_materialize_alias_tbl(" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java index eb3a42c3868610..d492100e867f29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java @@ -17,6 +17,12 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; @@ -27,7 +33,10 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate.PushDownAggOp; @@ -38,6 +47,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.Optional; @@ -115,6 +125,67 @@ public void testWithoutProject() { ); } + @Test + public void testNullableFileCountUsesStorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + LogicalFileScan fileScan = aggregate.child(); + + PlanChecker.from(MemoTestUtils.createCascadesContext(aggregate)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .matches(logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(fileScan.getOutput().get(0).getExprId()))))); + } + + @Test + public void testNullableFileCountDoesNotUseV1StorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + CascadesContext context = MemoTestUtils.createCascadesContext(aggregate); + context.getConnectContext().getSessionVariable().enableFileScannerV2 = false; + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + @Test + public void testMixedCountStarAndNullableFileCountDoesNotUseStorageLayerAggregate() { + LogicalAggregate nullableCount = newNullableFileCountAggregate(); + LogicalFileScan fileScan = nullableCount.child(); + LogicalAggregate mixedCount = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star"), + new Alias(new Count(fileScan.getOutput().get(0)), "count_nullable")), + true, Optional.empty(), fileScan); + + PlanChecker.from(MemoTestUtils.createCascadesContext(mixedCount)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + private LogicalAggregate newNullableFileCountAggregate() { + Column nullableColumn = new Column("value", Type.INT, true); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema()).thenReturn(ImmutableList.of(nullableColumn)); + Mockito.when(table.getName()).thenReturn("nullable_file_table"); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn("catalog"); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(table.getDatabase()).thenReturn(database); + LogicalFileScan fileScan = new LogicalFileScan(new RelationId(1), table, + ImmutableList.of("catalog", "db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + return new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(fileScan.getOutput().get(0)), "count")), + true, Optional.empty(), fileScan); + } + @Override public RulePromise defaultPromise() { return RulePromise.IMPLEMENT; @@ -218,6 +289,15 @@ private Rule storageLayerAggregateWithoutProject() { .get(); } + private Rule storageLayerAggregateWithoutProjectForFileScan() { + return new AggregateStrategies().buildRules() + .stream() + .filter(rule -> rule.getRuleType() + == RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT_FOR_FILE_SCAN) + .findFirst() + .get(); + } + private Rule storageLayerAggregateWithProject() { return new AggregateStrategies().buildRules() .stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java index df9b2ae80c26f2..74d231d3154a25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TIcebergMergeSink; import org.apache.doris.thrift.TIcebergRewritableDeleteFileSet; +import com.google.common.collect.ImmutableMap; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; @@ -76,7 +77,7 @@ public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV @Test public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Exception { - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext()); sink.bindDataSink(Optional.empty()); @@ -88,7 +89,7 @@ public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Ex @Test public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exception { - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "counts")), new DeleteCommandContext()); @@ -102,7 +103,7 @@ public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exceptio @Test public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exception { - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, Map.of( + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, ImmutableMap.of( TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none")), new DeleteCommandContext()); @@ -118,7 +119,7 @@ public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exceptio public void testBindDataSinkKeepsColumnStatsForOrcTopLevelComplexField() throws Exception { Schema schema = new Schema(Types.NestedField.optional(1, "items", Types.ListType.ofOptional(2, Types.IntegerType.get()))); - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, schema, Map.of( + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, schema, ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "orc", TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "items", "counts")), diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 16640db7ccddfe..02356bb094efa6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -54,6 +54,8 @@ import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TStorageType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import mockit.Mock; @@ -91,31 +93,31 @@ void testGetIcebergColumnStatsReturnsEmptyForDisabledMetrics() { Mockito.when(table.schema()).thenReturn(schema); Mockito.when(table.spec()).thenReturn(spec); Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); - Mockito.when(table.properties()).thenReturn(Map.of( + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( TableProperties.DEFAULT_FILE_FORMAT, "parquet", TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); TIcebergColumnStats columnStats = new TIcebergColumnStats(); - columnStats.setColumnSizes(Map.of(1, 128L)); - columnStats.setValueCounts(Map.of(1, 10L)); - columnStats.setNullValueCounts(Map.of(1, 0L)); + columnStats.setColumnSizes(ImmutableMap.of(1, 128L)); + columnStats.setValueCounts(ImmutableMap.of(1, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(1, 0L)); TIcebergCommitData commitData = new TIcebergCommitData(); commitData.setFilePath("/path/to/data.parquet"); commitData.setRowCount(10); commitData.setFileSize(1024); commitData.setColumnStats(columnStats); - DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; TableScan tableScan = Mockito.mock(TableScan.class); FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); Mockito.when(table.newScan()).thenReturn(tableScan); Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); Mockito.when(tableScan.planFiles()) - .thenReturn(CloseableIterable.withNoopClose(List.of(fileScanTask))); + .thenReturn(CloseableIterable.withNoopClose(ImmutableList.of(fileScanTask))); Mockito.when(fileScanTask.spec()).thenReturn(spec); Mockito.when(fileScanTask.file()).thenReturn(dataFile); - Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + Assertions.assertTrue(!StatisticsUtil.getIcebergColumnStats("id", table).isPresent()); } @Test @@ -129,15 +131,15 @@ void testGetIcebergColumnStatsReturnsEmptyWhenCloseFails() { Mockito.when(table.newScan()).thenReturn(tableScan); Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); Mockito.when(tableScan.planFiles()).thenReturn(CloseableIterable.combine( - List.of(fileScanTask), () -> { + ImmutableList.of(fileScanTask), () -> { throw new IOException("close failed"); })); Mockito.when(fileScanTask.spec()).thenReturn(spec); Mockito.when(fileScanTask.file()).thenReturn(dataFile); - Mockito.when(dataFile.columnSizes()).thenReturn(Map.of()); - Mockito.when(dataFile.nullValueCounts()).thenReturn(Map.of()); + Mockito.when(dataFile.columnSizes()).thenReturn(ImmutableMap.of()); + Mockito.when(dataFile.nullValueCounts()).thenReturn(ImmutableMap.of()); - Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + Assertions.assertTrue(!StatisticsUtil.getIcebergColumnStats("id", table).isPresent()); } @Test diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index cf7028d996f3d7..8ae188a73118f0 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -253,9 +253,21 @@ struct TFileTextScanRangeParams { 8: optional bool empty_field_as_null } +enum TColumnCategory { + REGULAR = 0, + PARTITION_KEY = 1, + SYNTHESIZED = 2, + GENERATED = 3, +} + struct TFileScanSlotInfo { 1: optional Types.TSlotId slot_id; 2: optional bool is_file_slot; + 3: optional TColumnCategory category; + // Default value expression for this column when it is missing from the data file. + // Populated by FE from Column.getDefaultValue() or NULL literal. + // This replaces the separate default_value_of_src_slot map in TFileScanRangeParams. + 4: optional Exprs.TExpr default_value_expr; } // descirbe how to read file @@ -436,6 +448,8 @@ struct TTableFormatFileDesc { 8: optional TLakeSoulFileDesc lakesoul_params 9: optional i64 table_level_row_count = -1 10: optional TRemoteDorisFileDesc remote_doris_params + // JDBC connection parameters (used when table_format_type == "jdbc") + 11: optional map jdbc_params } // Deprecated, hive text talbe is a special format, not a serde type diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 4fe42d7fcdcc77..bb74c79d5a6907 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -808,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -866,10 +866,10 @@ abcDeFGhijkLmnOp \N 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index fbb0e343e09039..52cd0c60ad36f0 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -177,25 +177,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("2", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("2", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("1", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("1", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -208,25 +208,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -239,25 +239,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -271,26 +271,26 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("8", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("8", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("7", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("7", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("6", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("6", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("9", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("9", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } From c8fba22e2452463b7bdd1dd2d404b1c77cd7016d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 14:52:11 +0800 Subject: [PATCH 16/24] [fix](test) Stabilize timing-sensitive FE unit tests --- .../cloud/CacheHotspotManagerTableFilterTest.java | 10 ++++++---- .../apache/doris/common/profile/AutoProfileTest.java | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java index 1b1742ece8a6a8..2bada7864d3642 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java @@ -905,8 +905,9 @@ public void testShouldWarmUpPerformance200kTables() { Assertions.assertEquals(200000, matched); System.out.println("[Perf] 200K tables, wildcard match-all: " + elapsedMs + " ms"); - Assertions.assertTrue(elapsedMs < 1500, - "200K regex matches should complete within 1.5s, took " + elapsedMs + " ms"); + // Shared FE UT workers need a broad guardrail; exact throughput belongs in dedicated benchmarks. + Assertions.assertTrue(elapsedMs < 3000, + "200K regex matches should complete within 3s, took " + elapsedMs + " ms"); } @Test @@ -1041,8 +1042,9 @@ public void testShouldWarmUpPerformanceRepeatedCycles200k() { Assertions.assertEquals(200000 * iterations, totalMatched); System.out.println("[Perf] 200K tables × 5 cycles: total=" + totalMs + " ms, avg=" + avgMs + " ms/cycle"); - Assertions.assertTrue(avgMs < 1000, - "Avg per refresh cycle for 200K tables should be < 1s, avg=" + avgMs + " ms"); + // Shared FE UT workers need a broad guardrail; exact throughput belongs in dedicated benchmarks. + Assertions.assertTrue(avgMs < 2000, + "Avg per refresh cycle for 200K tables should be < 2s, avg=" + avgMs + " ms"); } private static class RecordingAppender extends AbstractAppender { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java index 5475cfc4c98e8b..67209d224bd963 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java @@ -64,15 +64,16 @@ public void testAutoProfile() throws InterruptedException { result = System.currentTimeMillis(); } }; - profile.autoProfileDurationMs = 1000; - Thread.sleep(899); + // Keep ample margins because CI scheduling delays must not decide which threshold branch is exercised. + profile.autoProfileDurationMs = 10000; + Thread.sleep(100); profile.updateSummary(summaryInfo, true, null); Assertions.assertNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId())); profile = createProfile(); profile.setSummaryProfile(summaryProfile); - profile.autoProfileDurationMs = 500; - Thread.sleep(899); + profile.autoProfileDurationMs = 50; + Thread.sleep(200); profile.updateSummary(summaryInfo, true, null); Assertions.assertNotNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId())); } From 01c4b2da57ad71872f57082bda1aa65db3da77b2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 15:18:05 +0800 Subject: [PATCH 17/24] [fix](test) Preserve Iceberg split format coverage after rebase --- .../iceberg/source/IcebergScanNodeTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 731f7f860fdbc8..504f0f48d090f8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -539,6 +539,26 @@ public void testSetIcebergParamsKeepsDeletionVectorOffsetAsLong() throws Excepti Assert.assertEquals((long) Integer.MAX_VALUE + 7L, deleteFileDesc.getContentSizeInBytes()); } + @Test + public void testSetIcebergParamsUsesSplitFileFormat() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + String dataPath = "file:///tmp/data-file.orc"; + IcebergSplit split = new IcebergSplit(LocationPath.of(dataPath), 0, 128, 128, new String[0], + 2, Collections.emptyMap(), new ArrayList<>(), dataPath); + split.setTableFormatType(TableFormatType.ICEBERG); + split.setSplitFileFormat(FileFormat.ORC); + + Method method = IcebergScanNode.class.getDeclaredMethod("setIcebergParams", + TFileRangeDesc.class, IcebergSplit.class); + method.setAccessible(true); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + method.invoke(node, rangeDesc, split); + + // Iceberg tables may mix file formats, so each range must preserve its split format. + Assert.assertEquals(TFileFormatType.FORMAT_ORC, rangeDesc.getFormatType()); + } + @Test public void testPositionDeleteSystemTableValidatesDeletionVectorMetadata() throws Exception { DeleteFile deleteFile = Mockito.mock(DeleteFile.class); From a943f4bbe32746a60a0c5e5e33b9cabebbf75964 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 08:01:50 +0800 Subject: [PATCH 18/24] [bench](parquet) Add native reader microbenchmark matrix (#65921) Adds a reproducible local microbenchmark foundation for the native Parquet reader described in the [benchmark design]. It separates page decoder cost from end-to-end local reader cost and keeps fixture construction and encoding validation outside measured regions. - Add 152 native decoder cases covering PLAIN, dictionary, byte-stream-split, and DELTA encodings across fixed-width and binary physical types, with clustered and alternating sparse selections. - Add 137 local reader cases covering open-to-first-block, full scan, predicate scan, LIMIT shapes, null density/pattern, predicate selectivity, projection mode, schema width, predicate position, and four physical encodings. - Generate deterministic fixtures lazily and validate every row group and column encoding from Parquet footer metadata before measuring. - Report rows/s, bytes/s, raw/selected rows, fixture size, ns/raw row, and ns/selected row. - Add matrix constraint tests and local build/run documentation. - [x] Release benchmark target builds with `ninja -C be/build_RELEASE -j128 benchmark_test` - [x] Scenario matrix tests pass (4/4) - [x] All 152 decoder benchmarks pass - [x] All 137 reader benchmarks pass - [x] clang-format 16 and `git diff --check` pass (cherry picked from commit 9ec7f76d15efc2df544b8ba6c1afdee76fb9d8c4) --- be/benchmark/benchmark_main.cpp | 87 +++- be/benchmark/parquet/AGENTS.md | 296 +++++++++++ be/benchmark/parquet/README.md | 63 +++ .../parquet/benchmark_parquet_decoder.hpp | 479 ++++++++++++++++++ .../parquet/benchmark_parquet_reader.hpp | 448 ++++++++++++++++ .../parquet/parquet_benchmark_scenarios.h | 250 +++++++++ .../parquet_benchmark_scenarios_test.cpp | 19 - 7 files changed, 1619 insertions(+), 23 deletions(-) create mode 100644 be/benchmark/parquet/AGENTS.md create mode 100644 be/benchmark/parquet/README.md create mode 100644 be/benchmark/parquet/benchmark_parquet_decoder.hpp create mode 100644 be/benchmark/parquet/benchmark_parquet_reader.hpp create mode 100644 be/benchmark/parquet/parquet_benchmark_scenarios.h diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index ff57d89c802d77..582db25ff6d667 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -17,22 +17,77 @@ #include +#include +#include +#include +#include + #include "benchmark_arrow_validation.hpp" #include "benchmark_bit_pack.hpp" +#include "benchmark_bits.hpp" +#include "benchmark_block_bloom_filter.hpp" #include "benchmark_column_array_view.hpp" #include "benchmark_column_array_view_distance.hpp" +#include "benchmark_column_view.hpp" +#include "benchmark_damerau_levenshtein.hpp" #include "benchmark_fastunion.hpp" #include "benchmark_fmod.hpp" #include "benchmark_hll_merge.hpp" +#include "benchmark_hybrid_set.hpp" +#include "benchmark_pdep_unpack.hpp" +#include "benchmark_string.hpp" +#include "benchmark_string_replace.hpp" #include "benchmark_zone_map_index.hpp" #include "binary_cast_benchmark.hpp" +#include "common/config.h" #include "core/block/block.h" -#include "vec/columns/column_string.h" -#include "vec/data_types/data_type.h" -#include "vec/data_types/data_type_string.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_string.h" +#include "parquet/benchmark_parquet_decoder.hpp" +#include "parquet/benchmark_parquet_reader.hpp" +#include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/thread_context.h" + +// benchmark_binary_plain_page_v2.hpp must be included LAST: it transitively pulls AWS SDK +// headers (via storage/cache/page_cache.h) whose symbols shadow types used by the benchmark +// headers above (notably binary_cast_benchmark.hpp). Keeping it last avoids the clash without +// disabling any benchmark. (Do not let clang-format reorder it above the others.) +#include "benchmark_binary_plain_page_v2.hpp" namespace doris { // change if need +static bool init_benchmark_config(const char* executable) { + std::vector candidates; + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + candidates.emplace_back(std::filesystem::path(doris_home) / "conf" / "be.conf"); + } + candidates.emplace_back(std::filesystem::current_path() / "conf" / "be.conf"); + const auto executable_dir = std::filesystem::absolute(executable).parent_path(); + candidates.emplace_back(executable_dir / ".." / ".." / "conf" / "be.conf"); + candidates.emplace_back(executable_dir / ".." / ".." / ".." / "conf" / "be.conf"); + for (const auto& candidate : candidates) { + if (!std::filesystem::exists(candidate)) { + continue; + } + if (std::getenv("DORIS_HOME") == nullptr) { + const auto inferred_home = candidate.parent_path().parent_path().string(); + // be.conf contains paths relative to DORIS_HOME. Keep standalone benchmark launches + // equivalent to build/test scripts after locating the same repository config. + if (::setenv("DORIS_HOME", inferred_home.c_str(), 0) != 0) { + return false; + } + } + // Mutable config storage is populated by config::init. Reader benchmarks must use the + // production defaults because zero-initialized safety limits reject every valid footer. + return config::init(candidate.c_str(), false); + } + std::cerr << "Unable to find conf/be.conf for benchmark initialization\n"; + return false; +} + static void Example1(benchmark::State& state) { // init. dont time it. state.PauseTiming(); @@ -55,4 +110,28 @@ static void Example1(benchmark::State& state) { BENCHMARK(Example1); } // namespace doris -BENCHMARK_MAIN(); +// Custom main: benchmarks that touch DataPage allocation require a Doris +// ThreadContext + mem tracker, otherwise the allocator throws E-7412. Mirrors +// the minimal subset of be/test/testutil/run_all_tests.cpp::main. +int main(int argc, char** argv) { + if (!doris::init_benchmark_config(argv[0])) { + return 1; + } + doris::config::enable_bmi2_optimizations = true; + + SCOPED_INIT_THREAD_CONTEXT(); + doris::ExecEnv::GetInstance()->init_mem_tracker(); + doris::thread_context()->thread_mem_tracker_mgr->init(); + auto bench_tracker = doris::MemTrackerLimiter::create_shared( + doris::MemTrackerLimiter::Type::GLOBAL, "BE-BENCH"); + doris::thread_context()->thread_mem_tracker_mgr->attach_limiter_tracker(bench_tracker); + doris::ExecEnv::set_tracking_memory(false); + + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + return 0; +} diff --git a/be/benchmark/parquet/AGENTS.md b/be/benchmark/parquet/AGENTS.md new file mode 100644 index 00000000000000..a8667fb43cdf5f --- /dev/null +++ b/be/benchmark/parquet/AGENTS.md @@ -0,0 +1,296 @@ +# Parquet microbenchmark guide for agents + +This file applies to `be/benchmark/parquet/`. Read it before changing, running, or interpreting +these benchmarks. The current suite is a local benchmark foundation, not the complete Parquet +benchmark system described in the design document. + +## What exists today + +The benchmark binary registers two groups: + +- `ParquetDecoder`: native page decoder benchmarks using in-memory encoded pages. +- `ParquetReader`: local-file benchmarks that call the format V2 Parquet reader directly. + +The relevant files are: + +- `benchmark_parquet_decoder.hpp`: deterministic page construction and decoder registration. +- `benchmark_parquet_reader.hpp`: deterministic local Parquet fixtures and reader registration. +- `parquet_benchmark_scenarios.h`: scenario definitions and the selected matrix. +- `README.md`: short human-oriented build and invocation examples. +- `be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp`: matrix invariants. + +Do not describe this suite as end-to-end SQL, `FileScannerV2`, remote I/O, V1/V2 comparison, or a +cross-engine benchmark. The reader benchmark starts at `format::parquet::ParquetReader` and does +not include FE planning, scanner scheduling, `TableReader`, client latency, or Runtime Profile +collection. + +## Build and list cases + +Performance results must come from a Release build: + +```shell +./build.sh --benchmark -j128 +``` + +List all Parquet cases and verify the expected registration counts: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -E '^Parquet(Decoder|Reader)/' + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetDecoder/' # currently 152 + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetReader/' # currently 137 +``` + +When running the binary directly from `be/build_RELEASE/bin`, make sure the JVM and third-party +libraries are discoverable. Prefer the installed `be/output/lib/benchmark_test` produced by +`build.sh` because the normal Doris environment already configures its dependencies. + +## Run smoke verification + +A smoke run proves that every registered case initializes and completes. It does not produce a +stable performance baseline: + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-decoder-smoke.json \ + --benchmark_out_format=json + +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-reader-smoke.json \ + --benchmark_out_format=json +``` + +Reject a smoke run if the process is non-zero, the expected number of JSON results is absent, or +any result contains `error_occurred`. Also run the scenario matrix unit test after changing the +matrix. Never use the 1 ms smoke measurements to claim a speedup or regression. + +## Run a performance comparison + +Compare the same named cases on the same host, compiler, build flags, CPU set, NUMA node, and power +policy. Use the same fixture directory and cache state for both revisions. A suitable local command +is: + +```shell +taskset -c 8 be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=1s \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Use at least three untimed warmups before collecting local warm-cache results. Run revisions in an +ABBA order when comparing two commits. Do not compare results from different CPUs, cache +topologies, compiler versions, or build types. + +The suite does not currently control the OS page cache. Repeated reader cases are normally +warm-cache measurements. Do not label them cold-NVMe results. Do not clear a shared machine's page +cache to manufacture a cold run. + +## Current scenario matrix + +`ParquetDecoder` contains 19 encoding/type pairs. Each pair is run at 1%, 10%, 50%, and 100% +selection with clustered and alternating selection ranges, for 152 registered cases. + +| Encoding | Physical types | +|---|---| +| PLAIN | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| Dictionary | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| BYTE_STREAM_SPLIT | FLOAT, DOUBLE, FIXED_LEN_BYTE_ARRAY | +| DELTA_BINARY_PACKED | INT32, INT64 | +| DELTA_LENGTH_BYTE_ARRAY | BYTE_ARRAY | +| DELTA_BYTE_ARRAY | BYTE_ARRAY | + +`ParquetReader` deliberately uses a single-variable matrix rather than a Cartesian product. After +deduplication it contains 137 cases covering: + +- operations: open-to-first-block, full scan, predicate scan, limit 1, and limit 1000; +- file encodings: PLAIN, dictionary, BYTE_STREAM_SPLIT, and DELTA_BINARY_PACKED; +- null ratios: 0%, 1%, 10%, 50%, and 90%; +- null shapes: clustered and alternating; +- predicate selectivities: 0%, 1%, 10%, 50%, 90%, and 100%; +- projection shapes: predicate-only and predicate plus one lazy payload column; +- schema widths: 4, 32, 128, and 512 columns; +- predicate position: first or last column. + +Except for the axis being varied, reader cases inherit the baseline: nullable INT32, PLAIN, +alternating 10% nulls, 10% selectivity, 32 columns, predicate at column zero, and predicate plus +payload projection. + +## How decoder data is generated + +Decoder pages are constructed in memory before the timed loop. There is no Parquet file, Python +generator, random seed, or manifest involved. + +- Every page contains 65,536 logical values. +- Integer values use `(row * 17) % 1000003`. +- Floating-point values use `(row % 1009) * 0.25 - 100.0`. +- Binary values are fixed at 16 bytes. Their first bytes contain `row % 1009` and the remaining + bytes are deterministic filler. +- PLAIN fixed-width values are copied in physical byte order. PLAIN BYTE_ARRAY values use a + four-byte length followed by payload bytes. +- BYTE_STREAM_SPLIT pages transpose the byte lanes of the fixed-width PLAIN representation. +- DELTA pages are produced with the Arrow Parquet DELTA encoders. +- Dictionary pages contain 256 deterministic entries. IDs repeat from 0 through 255 and are + encoded with an eight-bit RLE/bit-packed stream. +- Clustered selection is one continuous range. Alternating selection spreads selected rows evenly + and intentionally produces many one-row physical ranges. + +Page generation, selection construction, decoder creation, dictionary setup, and `set_data` are +outside the timed decode call. The sinks consume decoder callbacks and prevent compiler removal, +but they do not build a Doris `Column`. Consequently these cases isolate decoder traversal and +selection cost; they do not measure definition-level decoding, nullable reconstruction, type +conversion, or full column materialization. + +## How reader Parquet files are generated + +Reader fixtures are generated lazily by C++ code using Arrow builders and the Arrow Parquet writer. +They are stored under: + +```text +${TMPDIR:-/tmp}/doris_parquet_reader_benchmark/ +``` + +Fixture contents and writer settings are: + +- 16,384 rows and 4 row groups of 4,096 rows each; +- all columns are nullable INT32 and reuse the same deterministic Arrow array; +- every non-null value is `row % 100`; +- the predicate is `value < selectivity_percent`, so the threshold maps directly to the intended + non-null selectivity; +- alternating nulls use a 101-row period and `(row * 37) % 101`, avoiding direct correlation with + the 100-value predicate period; +- clustered nulls use contiguous null prefixes inside each 1,024-row cluster; +- Parquet format version 2.6, DataPage V2, no compression, and statistics disabled; +- dictionary is explicitly enabled only for dictionary fixtures; other fixtures disable dictionary + and request their named encoding; +- after writing, every column chunk in every row group is checked through footer metadata to ensure + that the writer did not silently choose another encoding. + +The fixture name depends only on axes that change file contents. Therefore predicate thresholds, +projection mode, and operation can reuse the same file. Existing fixtures are footer-validated +before reuse. Delete only the dedicated `doris_parquet_reader_benchmark` temporary directory when +fixture rules change; never remove a broad temporary or workspace directory. + +Fixture creation and footer verification happen before benchmark iterations. Reader initialization +and `close()` are excluded from steady-state full/predicate/LIMIT timing. The +`open_to_first_block` case intentionally includes reader construction, metadata loading, `open()`, +and the first `get_block()` call. + +## Interpret the result correctly + +Google Benchmark JSON contains `real_time`, `cpu_time`, repetitions, and custom counters. Use: + +- `cpu_time` for decoder CPU efficiency; +- both `real_time` and `cpu_time` for reader cases, because latency and CPU cost answer different + questions; +- median across repetitions as the primary result; +- p95 and a confidence interval for automated gates; never select the fastest repetition. + +Custom counters mean: + +- `raw_rows`: logical source rows considered by the case; +- `selected_rows`: rows returned after the predicate or selection; +- `ns/raw_row`: normalized source-row cost and the primary comparison across selectivities; +- `ns/selected_row`: cost per surviving row; it naturally rises as selectivity falls; +- `selection_ranges`: decoder physical ranges; compare alternating with clustered at the same + selectivity to expose per-range overhead; +- `fixture_bytes` or `encoded_bytes`: fixture/page size, not peak memory; +- `items_per_second`: selected rows per second, so it is not directly comparable across different + selectivities; +- `bytes_per_second`: a logical benchmark counter. In reader cases it is estimated from projected + INT32 values, not measured storage-device bytes or compressed bytes. + +Useful comparisons are: + +- clustered versus alternating at equal selectivity: sparse-range/cursor overhead; +- null 0% versus 1/10/50/90% at equal selection: definition-level and nullable reconstruction + overhead in the reader; +- predicate-only versus predicate-projected: lazy materialization benefit; +- width 4/32/128/512 with predicate first/last: schema and metadata traversal overhead; +- PLAIN versus dictionary/BYTE_STREAM_SPLIT/DELTA at the same reader baseline: encoding path cost; +- open-to-first-block and LIMIT cases separately from steady-state full scans. + +Before attributing a regression, confirm that output row counts are identical, the benchmark name +and fixture encoding match, and repetition variability is acceptable. A coefficient of variation +above 3% is inconclusive and should be rerun. Correlate CPU regressions with `perf stat` counters +such as cycles, instructions, branch misses, and cache misses when possible. + +## Coverage audit and required follow-up + +The current matrix is not comprehensive. Preserve this distinction in PR descriptions and reports. + +### P0: complete the local benchmark phase + +1. Add the design's `matrix.yaml`, deterministic corpus generator, `manifest.json`, checksum, and + standalone corpus verifier. The current runtime-generated files cannot be shared unchanged with + V1, StarRocks, or DuckDB. +2. Add correctness oracles. Benchmarks must validate consumed counts and representative checksums + outside the timed region before their performance samples are trusted. +3. Add reader-level INT64, FLOAT, DOUBLE, BYTE_ARRAY/string, FIXED_LEN_BYTE_ARRAY, DATE, + TIMESTAMP, and DECIMAL cases. Today only nullable INT32 reaches the complete reader path. +4. Extend decoder coverage with 0% and 90% selection, definition levels/null reconstruction, + dictionary conversion, and real Doris `Column` materialization. The current decoder sink does + not cover those costs or report decoded bytes per second. +5. Add the representative nullable sparse corpus requested by the design: 32 INT64 columns, 128 + row groups, and enough rows to exercise many pages. The current 16K-row/four-row-group fixture + is a smoke-sized workload. +6. Add dictionary footprint sweeps based on actual bytes relative to L1/L2 cache, value-width + sweeps, cardinality sweeps, and dictionary-to-PLAIN fallback/mixed-page files. The current + dictionary is small and fixed. +7. Add a Doris V1/V2 SQL runner over the same verified corpus, correctness comparison, Runtime + Profile collection, normalized result JSON, comparison report, and the first stable local + baseline. The current 1 ms run only proves executability. + +### P1: broaden Parquet format and execution behavior + +1. Vary compression codecs: uncompressed, Snappy, ZSTD, LZ4, Brotli, and Gzip where supported. +2. Cover DataPage V1 and V2, page size, row-group size/count, page boundaries, small files, and + large files that exceed cache capacity. +3. Add required/non-nullable columns, correlated and anti-correlated null/predicate distributions, + random deterministic runs, long null runs, and selections that cross page boundaries. +4. Add Boolean, INT96 compatibility, logical annotations, timestamp units/time zones, decimal + physical representations, short/long strings, skew, and empty values. +5. Add nested ARRAY/MAP/STRUCT and repeated definition/repetition-level cases. +6. Add row-group statistics, page index, Bloom filter, dictionary filtering, all-filtered batches, + missing-column/default-value handling, schema mapping, and schema-evolution cases. Keep pruning + benchmarks separate from decode benchmarks so skipped work is not misattributed to a faster + decoder. +7. Add predicate LIMIT 1/1000 cases. Current LIMIT cases only vary reader batch size without a + predicate. +8. Add peak tracked memory, allocation counts, and an explicit warm/cold-cache methodology. + +### Later phases + +Remote S3/FileCache/RTT workloads, prefetch metrics, cross-engine adapters, nightly/weekly jobs, and +release regression gates belong to later phases. They require isolated infrastructure and must not +be simulated by silently changing the local reader benchmark. + +## Current validation record + +At commit `16e05dd5c71`, a Release build completed and the matrix unit test passed 4/4. A 1 ms smoke +run executed 152 decoder and 137 reader cases with zero benchmark errors. This is an execution +record only. It is not a reviewed performance baseline because repetitions, host isolation, +warmups, cache control, `perf` data, variance, and before/after comparison were not collected. + +## Rules for extending the suite + +- Keep deterministic data construction and validation outside timed regions. +- Verify actual footer encodings; never trust writer configuration alone. +- Change one primary axis at a time and add only meaningful second-order interactions. +- Add or update matrix invariant tests before registering new dimensions. +- Keep correctness/error-injection tests separate from throughput benchmarks. +- Do not commit large generated corpora. Commit generation rules and manifests; keep only small + correctness fixtures in the repository when necessary. +- Record the exact commit, compiler, build type, host topology, command, repetitions, cache state, + and raw JSON for any claimed performance result. +- Never claim a performance improvement from a smoke run or from incomparable machines. diff --git a/be/benchmark/parquet/README.md b/be/benchmark/parquet/README.md new file mode 100644 index 00000000000000..40c79bc28bea5f --- /dev/null +++ b/be/benchmark/parquet/README.md @@ -0,0 +1,63 @@ +# Parquet reader microbenchmarks + +These benchmarks separate native page decoding from the complete local-file reader path. They use +deterministic data and verify the physical encoding recorded in each generated Parquet footer before +running a measurement. + +Agents and maintainers must read [AGENTS.md](AGENTS.md) for the exact timing boundaries, synthetic +data rules, result interpretation, current coverage limitations, and prioritized follow-up work. + +## Build + +Build the Release benchmark binary from the repository root: + +```shell +./build.sh --benchmark -j128 +``` + +List only the Parquet cases: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests | grep '^Parquet' +``` + +## Decoder cases + +`ParquetDecoder` measures the native decoder with data generation and encoder setup outside the +timed region. It covers PLAIN, dictionary, byte-stream-split, and DELTA encodings across their +supported fixed-width and binary physical types. Sparse selections are provided as both one +clustered range and many alternating ranges. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/plain/int64/sel_10/alternating$' \ + --benchmark_min_time=0.1s +``` + +## Local reader cases + +`ParquetReader` measures local open-to-first-block, full scan, predicate scan, and LIMIT-shaped +reads. The matrix covers: + +- PLAIN, dictionary, byte-stream-split, and DELTA binary-packed files; +- NULL ratios of 0%, 1%, 10%, 50%, and 90%, with clustered and alternating placement; +- predicate selectivities of 0%, 1%, 10%, 50%, 90%, and 100%; +- predicate-only and predicate-plus-lazy-projected reads; +- schemas with 4, 32, 128, and 512 columns, with the predicate first or last. + +Fixtures are created lazily under the system temporary directory in +`doris_parquet_reader_benchmark`. Generation, footer validation, and reader setup are excluded from +steady-state scan timings. `open_to_first_block` intentionally includes reader initialization, +footer loading, open, and the first `get_block` call. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=0.1s \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Every result reports throughput plus `raw_rows`, `selected_rows`, `fixture_bytes`, `ns/raw_row`, +and (when at least one row survives) `ns/selected_row`. Keep CPU frequency, build type, compiler, +machine placement, and benchmark filters fixed when comparing two commits. diff --git a/be/benchmark/parquet/benchmark_parquet_decoder.hpp b/be/benchmark/parquet/benchmark_parquet_decoder.hpp new file mode 100644 index 00000000000000..25fb05d1449e53 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_decoder.hpp @@ -0,0 +1,479 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "parquet_benchmark_scenarios.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::parquet_benchmark { +namespace detail { + +constexpr size_t DECODER_ROWS = 1UL << 16; +constexpr size_t DICTIONARY_ENTRIES = 256; +constexpr size_t FIXED_BINARY_WIDTH = 16; + +struct EncodedPage { + std::vector data; + std::vector dictionary; + size_t dictionary_entries = 0; + size_t value_width = 0; + bool binary = false; +}; + +inline tparquet::Type::type physical_type(ValueType value_type) { + switch (value_type) { + case ValueType::INT32: + return tparquet::Type::INT32; + case ValueType::INT64: + return tparquet::Type::INT64; + case ValueType::FLOAT: + return tparquet::Type::FLOAT; + case ValueType::DOUBLE: + return tparquet::Type::DOUBLE; + case ValueType::BYTE_ARRAY: + return tparquet::Type::BYTE_ARRAY; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return tparquet::Type::FIXED_LEN_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark value type"); +} + +inline tparquet::Encoding::type parquet_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return tparquet::Encoding::PLAIN; + case Encoding::DICTIONARY: + return tparquet::Encoding::RLE_DICTIONARY; + case Encoding::BYTE_STREAM_SPLIT: + return tparquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return tparquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return tparquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type type, + int type_length = -1) { + auto node = type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY + ? ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type, + ::parquet::ConvertedType::NONE, type_length) + : ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +template +std::vector fixed_values(size_t rows) { + std::vector values(rows); + for (size_t row = 0; row < rows; ++row) { + if constexpr (std::is_floating_point_v) { + values[row] = static_cast((row % 1009) * 0.25 - 100.0); + } else { + values[row] = static_cast((row * 17) % 1000003); + } + } + return values; +} + +inline std::vector binary_values(size_t rows, size_t width = FIXED_BINARY_WIDTH) { + std::vector values; + values.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + std::string value(width, 'a'); + const uint64_t id = row % 1009; + memcpy(value.data(), &id, std::min(width, sizeof(id))); + values.push_back(std::move(value)); + } + return values; +} + +inline std::vector encode_plain_binary(const std::vector& values) { + size_t bytes = 0; + for (const auto& value : values) { + bytes += sizeof(uint32_t) + value.size(); + } + std::vector encoded; + encoded.reserve(bytes); + for (const auto& value : values) { + const auto length = static_cast(value.size()); + const auto* length_bytes = reinterpret_cast(&length); + encoded.insert(encoded.end(), length_bytes, length_bytes + sizeof(length)); + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +template +std::vector encode_plain_fixed(const std::vector& values) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + return encoded; +} + +inline std::vector encode_fixed_binary(const std::vector& values) { + std::vector encoded; + encoded.reserve(values.size() * FIXED_BINARY_WIDTH); + for (const auto& value : values) { + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +inline std::vector encode_byte_stream_split(const std::vector& plain, + size_t value_width) { + const size_t rows = plain.size() / value_width; + std::vector encoded(plain.size()); + for (size_t row = 0; row < rows; ++row) { + for (size_t byte = 0; byte < value_width; ++byte) { + encoded[byte * rows + row] = plain[row * value_width + byte]; + } + } + return encoded; +} + +template +std::vector encode_delta_fixed(const std::vector& values) { + auto desc = descriptor(ParquetType::type_num); + auto encoder = ::parquet::MakeTypedEncoder( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, desc.get()); + encoder->Put(values.data(), static_cast(values.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_delta_binary(const std::vector& values, + ::parquet::Encoding::type encoding) { + std::vector<::parquet::ByteArray> arrays; + arrays.reserve(values.size()); + for (const auto& value : values) { + arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto desc = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = + ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, desc.get()); + encoder->Put(arrays.data(), static_cast(arrays.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_dictionary_indices(size_t rows) { + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 8); + for (size_t row = 0; row < rows; ++row) { + encoder.Put(static_cast(row % DICTIONARY_ENTRIES)); + } + for (size_t padding = rows; padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector result(encoded_ids.size() + 1); + result[0] = 8; + memcpy(result.data() + 1, encoded_ids.data(), encoded_ids.size()); + return result; +} + +inline EncodedPage dictionary_page(ValueType value_type) { + EncodedPage page; + page.data = encode_dictionary_indices(DECODER_ROWS); + page.dictionary_entries = DICTIONARY_ENTRIES; + page.binary = value_type == ValueType::BYTE_ARRAY; + switch (value_type) { + case ValueType::INT32: + page.value_width = sizeof(int32_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::INT64: + page.value_width = sizeof(int64_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FLOAT: + page.value_width = sizeof(float); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::DOUBLE: + page.value_width = sizeof(double); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_plain_binary(binary_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FIXED_LEN_BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_fixed_binary(binary_values(DICTIONARY_ENTRIES)); + break; + } + return page; +} + +inline EncodedPage encoded_page(const DecoderScenario& scenario) { + if (scenario.encoding == Encoding::DICTIONARY) { + return dictionary_page(scenario.value_type); + } + + EncodedPage page; + page.binary = scenario.value_type == ValueType::BYTE_ARRAY; + switch (scenario.value_type) { + case ValueType::INT32: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int32_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int32Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::INT64: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int64_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int64Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::FLOAT: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(float); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::DOUBLE: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(double); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::BYTE_ARRAY: { + auto values = binary_values(DECODER_ROWS); + page.value_width = FIXED_BINARY_WIDTH; + if (scenario.encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY); + } else if (scenario.encoding == Encoding::DELTA_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_BYTE_ARRAY); + } else { + page.data = encode_plain_binary(values); + } + break; + } + case ValueType::FIXED_LEN_BYTE_ARRAY: { + auto plain = encode_fixed_binary(binary_values(DECODER_ROWS)); + page.value_width = FIXED_BINARY_WIDTH; + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + } + return page; +} + +class FixedSink final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + benchmark::DoNotOptimize(value_width); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class BinarySink final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + consumed += num_values; + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + benchmark::DoNotOptimize(encoded_data); + benchmark::DoNotOptimize(payload_offsets); + benchmark::DoNotOptimize(value_offsets); + auto value_spans_data = value_spans.data(); + benchmark::DoNotOptimize(value_spans_data); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class DictionarySink final : public ParquetDictionaryValueConsumer { +public: + DictionarySink(const uint8_t* dictionary, size_t value_width) + : _dictionary(dictionary), _value_width(value_width) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + auto* value = _dictionary + indices[row] * _value_width; + benchmark::DoNotOptimize(value); + } + consumed += num_values; + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + auto* value = _dictionary + index * _value_width; + benchmark::DoNotOptimize(value); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; + +private: + const uint8_t* _dictionary; + const size_t _value_width; +}; + +inline ParquetSelection native_selection(const SelectionPlan& plan) { + ParquetSelection selection { + .total_values = plan.total_rows, .selected_values = plan.selected_rows, .ranges = {}}; + selection.ranges.reserve(plan.ranges.size()); + for (const auto& range : plan.ranges) { + selection.ranges.push_back({.first = range.first, .count = range.count}); + } + return selection; +} + +inline void run_decoder(benchmark::State& state, DecoderScenario scenario, int selectivity, + Pattern pattern) { + auto page = encoded_page(scenario); + const auto plan = make_selection_plan(DECODER_ROWS, selectivity, pattern); + const auto selection = native_selection(plan); + std::unique_ptr decoder; + auto status = format::parquet::native::Decoder::get_decoder( + physical_type(scenario.value_type), parquet_encoding(scenario.encoding), decoder); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + decoder->set_type_length(static_cast(page.value_width)); + decoder->set_expected_values(DECODER_ROWS); + + std::vector dictionary_for_sink; + if (scenario.encoding == Encoding::DICTIONARY) { + dictionary_for_sink = page.dictionary; + auto dictionary = make_unique_buffer(page.dictionary.size()); + memcpy(dictionary.get(), page.dictionary.data(), page.dictionary.size()); + status = decoder->set_dict(dictionary, static_cast(page.dictionary.size()), + page.dictionary_entries); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + } + + Slice encoded(page.data.data(), page.data.size()); + FixedSink fixed_sink; + BinarySink binary_sink; + DictionarySink dictionary_sink(dictionary_for_sink.data(), page.value_width); + for (auto _ : state) { + state.PauseTiming(); + status = decoder->set_data(&encoded); + fixed_sink.consumed = 0; + binary_sink.consumed = 0; + dictionary_sink.consumed = 0; + state.ResumeTiming(); + if (scenario.encoding == Encoding::DICTIONARY) { + status = decoder->decode_selected_dictionary_values(selection, dictionary_sink); + } else if (page.binary) { + status = decoder->decode_selected_binary_values(selection, binary_sink); + } else { + status = decoder->decode_selected_fixed_values(selection, fixed_sink); + } + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + break; + } + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(plan.selected_rows)); + state.SetBytesProcessed(static_cast(state.iterations()) * + static_cast(page.data.size())); + state.counters["raw_rows"] = static_cast(DECODER_ROWS); + state.counters["selected_rows"] = static_cast(plan.selected_rows); + state.counters["selection_ranges"] = static_cast(plan.ranges.size()); + state.counters["encoded_bytes"] = static_cast(page.data.size()); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(DECODER_ROWS), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (plan.selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(plan.selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } +} + +inline bool register_decoder_benchmarks() { + for (const auto& scenario : decoder_scenarios()) { + for (const int selectivity : {1, 10, 50, 100}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + const std::string name = "ParquetDecoder/" + to_string(scenario.encoding) + "/" + + to_string(scenario.value_type) + "/sel_" + + std::to_string(selectivity) + "/" + to_string(pattern); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_decoder(state, scenario, selectivity, pattern); + })->Unit(benchmark::kNanosecond); + } + } + } + return true; +} + +inline const bool DECODER_BENCHMARKS_REGISTERED = register_decoder_benchmarks(); + +} // namespace detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/benchmark_parquet_reader.hpp b/be/benchmark/parquet/benchmark_parquet_reader.hpp new file mode 100644 index 00000000000000..ae3e14c0f66ad9 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_reader.hpp @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "gen_cpp/Types_types.h" +#include "io/io_common.h" +#include "parquet_benchmark_scenarios.h" +#include "runtime/runtime_state.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris::parquet_benchmark { +namespace reader_detail { + +constexpr size_t READER_ROWS = 1UL << 14; +constexpr size_t READER_ROW_GROUP_ROWS = 1UL << 12; + +inline void throw_if_error(const Status& status) { + if (!status.ok()) { + throw std::runtime_error(status.to_string()); + } +} + +inline bool is_null_row(size_t row, int null_percent, Pattern pattern) { + if (null_percent <= 0) { + return false; + } + if (pattern == Pattern::ALTERNATING) { + constexpr size_t NULL_PERIOD = 101; + const size_t null_slots = (static_cast(null_percent) * NULL_PERIOD + 99) / 100; + return (row * 37) % NULL_PERIOD < null_slots; + } + constexpr size_t CLUSTER_ROWS = 1024; + return row % CLUSTER_ROWS < CLUSTER_ROWS * static_cast(null_percent) / 100; +} + +inline std::shared_ptr build_int32_array(int null_percent, Pattern pattern) { + arrow::Int32Builder builder; + PARQUET_THROW_NOT_OK(builder.Reserve(READER_ROWS)); + for (size_t row = 0; row < READER_ROWS; ++row) { + if (is_null_row(row, null_percent, pattern)) { + PARQUET_THROW_NOT_OK(builder.AppendNull()); + } else { + PARQUET_THROW_NOT_OK(builder.Append(static_cast(row % 100))); + } + } + return builder.Finish().ValueOrDie(); +} + +inline ::parquet::Encoding::type file_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return ::parquet::Encoding::PLAIN; + case Encoding::BYTE_STREAM_SPLIT: + return ::parquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return ::parquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DICTIONARY: + return ::parquet::Encoding::RLE_DICTIONARY; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::string fixture_name(const ReaderScenario& scenario) { + return "v2_" + to_string(scenario.encoding) + "_null" + std::to_string(scenario.null_percent) + + "_" + to_string(scenario.null_pattern) + "_w" + std::to_string(scenario.schema_width) + + "_p" + std::to_string(scenario.predicate_position) + ".parquet"; +} + +inline void verify_fixture_encoding(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto reader = ::parquet::ParquetFileReader::OpenFile(path.string(), false); + const auto metadata = reader->metadata(); + if (metadata->num_rows() != static_cast(READER_ROWS) || + metadata->num_columns() != scenario.schema_width) { + throw std::runtime_error("Parquet benchmark fixture has unexpected shape: " + + path.string()); + } + const auto expected = file_encoding(scenario.encoding); + for (int row_group = 0; row_group < metadata->num_row_groups(); ++row_group) { + for (int column = 0; column < metadata->num_columns(); ++column) { + const auto encodings = metadata->RowGroup(row_group)->ColumnChunk(column)->encodings(); + if (std::ranges::find(encodings, expected) == encodings.end()) { + throw std::runtime_error("Parquet benchmark fixture did not use " + + to_string(scenario.encoding) + + " encoding: " + path.string()); + } + } + } +} + +inline std::filesystem::path ensure_fixture(const ReaderScenario& scenario) { + static std::mutex fixture_mutex; + const auto directory = + std::filesystem::temp_directory_path() / "doris_parquet_reader_benchmark"; + const auto path = directory / fixture_name(scenario); + std::lock_guard guard(fixture_mutex); + if (std::filesystem::exists(path)) { + verify_fixture_encoding(path, scenario); + return path; + } + + std::filesystem::create_directories(directory); + const auto temporary_path = path.string() + ".tmp"; + std::filesystem::remove(temporary_path); + const auto values = build_int32_array(scenario.null_percent, scenario.null_pattern); + std::vector> fields; + std::vector> columns; + fields.reserve(scenario.schema_width); + columns.reserve(scenario.schema_width); + for (int column = 0; column < scenario.schema_width; ++column) { + fields.push_back(arrow::field("c" + std::to_string(column), arrow::int32(), true)); + columns.push_back(std::make_shared(values)); + } + const auto table = arrow::Table::Make(arrow::schema(std::move(fields)), std::move(columns)); + + const auto output_result = arrow::io::FileOutputStream::Open(temporary_path); + if (!output_result.ok()) { + throw std::runtime_error(output_result.status().ToString()); + } + const auto output = *output_result; + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + properties.disable_statistics(); + if (scenario.encoding == Encoding::DICTIONARY) { + properties.enable_dictionary(); + } else { + properties.disable_dictionary(); + properties.encoding(file_encoding(scenario.encoding)); + } + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + READER_ROW_GROUP_ROWS, properties.build())); + PARQUET_THROW_NOT_OK(output->Close()); + std::filesystem::rename(temporary_path, path); + verify_fixture_encoding(path, scenario); + return path; +} + +class Int32LessThanExpr final : public VExpr { +public: + Int32LessThanExpr(int column_id, int32_t upper_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _upper_bound(upper_bound) {} + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& values = assert_cast(nullable.get_nested_column()); + const auto& nulls = nullable.get_null_map_data(); + auto result = ColumnUInt8::create(count, 0); + auto& matches = result->get_data(); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + matches[row] = !nulls[input_row] && values.get_element(input_row) < _upper_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int column_id, + uint8_t* matches) const override { + DORIS_CHECK(column_id == _column_id); + DORIS_CHECK(value_width == sizeof(int32_t)); + const auto* typed_values = reinterpret_cast(values); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= typed_values[row] < _upper_bound; + } + return Status::OK(); + } + + bool can_evaluate_zonemap_filter() const override { return true; } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto upper_bound = Field::create_field(_upper_bound); + return zone_map->min_value >= upper_bound ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto upper_bound = Field::create_field(_upper_bound); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value < upper_bound; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _upper_bound; + const std::string _expr_name = "ParquetBenchmarkInt32LessThan"; +}; + +inline VExprContextSPtr make_predicate(int column_position, int selectivity_percent) { + auto context = VExprContext::create_shared( + std::make_shared(column_position, selectivity_percent)); + context->_prepared = true; + context->_opened = true; + return context; +} + +inline Block make_block(const std::vector& schema) { + Block block; + for (const auto& column : schema) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +struct ReaderSession { + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + std::unique_ptr reader; + std::vector schema; + std::shared_ptr request; +}; + +inline std::unique_ptr open_reader(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto session = std::make_unique(); + auto properties = std::make_shared(); + properties->system_type = TFileType::FILE_LOCAL; + auto description = std::make_unique(); + description->path = path.string(); + description->file_size = static_cast(std::filesystem::file_size(path)); + description->range_start_offset = 0; + description->range_size = -1; + session->reader = std::make_unique(properties, description, + nullptr, nullptr); + throw_if_error(session->reader->init(&session->runtime_state)); + throw_if_error(session->reader->get_schema(&session->schema)); + + session->request = std::make_shared(); + format::FileScanRequestBuilder request_builder(session->request.get()); + if (scenario.operation == ReaderOperation::FULL_SCAN) { + for (int column = 0; column < scenario.schema_width; ++column) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(column))); + } + } else if (scenario.operation == ReaderOperation::PREDICATE_SCAN) { + const auto predicate_id = format::LocalColumnId(scenario.predicate_position); + throw_if_error(request_builder.add_predicate_column(predicate_id)); + if (scenario.projection == Projection::PREDICATE_ONLY) { + session->request->predicate_only_columns.push_back(predicate_id); + } else { + const int payload = scenario.predicate_position == 0 ? scenario.schema_width - 1 : 0; + throw_if_error( + request_builder.add_non_predicate_column(format::LocalColumnId(payload))); + } + const auto predicate_position = session->request->local_positions.at(predicate_id).value(); + session->request->conjuncts.push_back( + make_predicate(static_cast(predicate_position), scenario.selectivity_percent)); + } else { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(0))); + if (scenario.schema_width > 1) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(1))); + } + } + + if (scenario.operation == ReaderOperation::LIMIT_1) { + session->reader->set_batch_size(1); + } else if (scenario.operation == ReaderOperation::LIMIT_1000) { + session->reader->set_batch_size(1000); + } + throw_if_error(session->reader->open(session->request)); + return session; +} + +inline size_t scan_reader(ReaderSession* session, const ReaderScenario& scenario) { + size_t output_rows = 0; + bool eof = false; + const size_t limit = scenario.operation == ReaderOperation::LIMIT_1 ? 1 + : scenario.operation == ReaderOperation::LIMIT_1000 ? 1000 + : 0; + while (!eof) { + auto block = make_block(session->schema); + size_t rows = 0; + throw_if_error(session->reader->get_block(&block, &rows, &eof)); + output_rows += rows; + benchmark::DoNotOptimize(block); + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + break; + } + if (limit != 0 && output_rows >= limit) { + break; + } + } + return output_rows; +} + +inline size_t raw_rows_per_iteration(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::LIMIT_1) { + return 1; + } + if (scenario.operation == ReaderOperation::LIMIT_1000) { + return 1000; + } + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + return format::parquet::ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; + } + return READER_ROWS; +} + +inline int projected_columns(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::FULL_SCAN) { + return scenario.schema_width; + } + if (scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.projection == Projection::PREDICATE_ONLY) { + return 1; + } + return std::min(2, scenario.schema_width); +} + +inline void run_reader(benchmark::State& state, ReaderScenario scenario) { + std::filesystem::path fixture; + try { + fixture = ensure_fixture(scenario); + size_t selected_rows = 0; + for (auto _ : state) { + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.PauseTiming(); + } + auto session = open_reader(fixture, scenario); + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.ResumeTiming(); + } + selected_rows = scan_reader(session.get(), scenario); + state.PauseTiming(); + throw_if_error(session->reader->close()); + state.ResumeTiming(); + benchmark::ClobberMemory(); + } + + const auto raw_rows = raw_rows_per_iteration(scenario); + state.SetItemsProcessed(static_cast(state.iterations() * selected_rows)); + state.SetBytesProcessed(static_cast( + state.iterations() * raw_rows * projected_columns(scenario) * sizeof(int32_t))); + state.counters["raw_rows"] = static_cast(raw_rows); + state.counters["selected_rows"] = static_cast(selected_rows); + state.counters["fixture_bytes"] = static_cast(std::filesystem::file_size(fixture)); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(raw_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } + } catch (const std::exception& error) { + state.SkipWithError(error.what()); + } +} + +inline bool register_reader_benchmarks() { + for (const auto& scenario : reader_scenarios()) { + std::string name = + "ParquetReader/" + to_string(scenario.operation) + "/" + + to_string(scenario.encoding) + "/null_" + std::to_string(scenario.null_percent) + + "/" + to_string(scenario.null_pattern) + "/sel_" + + std::to_string(scenario.selectivity_percent) + "/" + + to_string(scenario.projection) + "/width_" + std::to_string(scenario.schema_width) + + "/predicate_" + std::to_string(scenario.predicate_position); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_reader(state, scenario); + })->Unit(benchmark::kNanosecond); + } + return true; +} + +inline const bool READER_BENCHMARKS_REGISTERED = register_reader_benchmarks(); + +} // namespace reader_detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/parquet_benchmark_scenarios.h b/be/benchmark/parquet/parquet_benchmark_scenarios.h new file mode 100644 index 00000000000000..d420d9ccdd552c --- /dev/null +++ b/be/benchmark/parquet/parquet_benchmark_scenarios.h @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::parquet_benchmark { + +enum class Encoding { + PLAIN, + DICTIONARY, + BYTE_STREAM_SPLIT, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY +}; +enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY }; +enum class Pattern { CLUSTERED, ALTERNATING }; +enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED }; +enum class ReaderOperation { OPEN_TO_FIRST_BLOCK, FULL_SCAN, PREDICATE_SCAN, LIMIT_1, LIMIT_1000 }; + +struct DecoderScenario { + Encoding encoding; + ValueType value_type; +}; + +struct ReaderScenario { + ReaderOperation operation; + Encoding encoding; + int null_percent; + Pattern null_pattern; + int selectivity_percent; + Projection projection; + int schema_width; + int predicate_position; +}; + +struct SelectionRange { + size_t first; + size_t count; +}; + +struct SelectionPlan { + size_t total_rows = 0; + size_t selected_rows = 0; + std::vector ranges; +}; + +inline std::vector decoder_scenarios() { + return { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; +} + +inline std::vector reader_scenarios() { + std::vector scenarios; + std::set> seen; + const auto add = [&](ReaderScenario scenario) { + const auto key = std::make_tuple(scenario.operation, scenario.encoding, + scenario.null_percent, scenario.null_pattern, + scenario.selectivity_percent, scenario.projection, + scenario.schema_width, scenario.predicate_position); + if (seen.insert(key).second) { + scenarios.push_back(scenario); + } + }; + + const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN, + .encoding = Encoding::PLAIN, + .null_percent = 10, + .null_pattern = Pattern::ALTERNATING, + .selectivity_percent = 10, + .projection = Projection::PREDICATE_PROJECTED, + .schema_width = 32, + .predicate_position = 0}; + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + auto scenario = baseline; + scenario.operation = operation; + add(scenario); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + auto scenario = baseline; + scenario.encoding = encoding; + add(scenario); + scenario.operation = ReaderOperation::PREDICATE_SCAN; + add(scenario); + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.schema_width = width; + scenario.predicate_position = predicate_position; + add(scenario); + } + } + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.null_percent = null_percent; + scenario.null_pattern = pattern; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } + } + return scenarios; +} + +inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent, + Pattern pattern) { + SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}}; + if (total_rows == 0 || selectivity_percent <= 0) { + return plan; + } + if (selectivity_percent >= 100) { + plan.selected_rows = total_rows; + plan.ranges.push_back({.first = 0, .count = total_rows}); + return plan; + } + plan.selected_rows = total_rows * static_cast(selectivity_percent) / 100; + if (plan.selected_rows == 0) { + plan.selected_rows = 1; + } + if (pattern == Pattern::CLUSTERED) { + plan.ranges.push_back({.first = 0, .count = plan.selected_rows}); + return plan; + } + + // Evenly spaced rows deliberately maximize the number of physical ranges. This is the + // adversarial sparse shape that exposes per-run decoder and cursor overhead. + for (size_t selected = 0; selected < plan.selected_rows; ++selected) { + const size_t row = selected * total_rows / plan.selected_rows; + if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) { + ++plan.ranges.back().count; + } else { + plan.ranges.push_back({.first = row, .count = 1}); + } + } + return plan; +} + +inline std::string to_string(Encoding value) { + switch (value) { + case Encoding::PLAIN: + return "plain"; + case Encoding::DICTIONARY: + return "dictionary"; + case Encoding::BYTE_STREAM_SPLIT: + return "byte_stream_split"; + case Encoding::DELTA_BINARY_PACKED: + return "delta_binary_packed"; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return "delta_length_byte_array"; + case Encoding::DELTA_BYTE_ARRAY: + return "delta_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(ValueType value) { + switch (value) { + case ValueType::INT32: + return "int32"; + case ValueType::INT64: + return "int64"; + case ValueType::FLOAT: + return "float"; + case ValueType::DOUBLE: + return "double"; + case ValueType::BYTE_ARRAY: + return "byte_array"; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return "fixed_len_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(Pattern value) { + return value == Pattern::CLUSTERED ? "clustered" : "alternating"; +} + +inline std::string to_string(Projection value) { + return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected"; +} + +inline std::string to_string(ReaderOperation value) { + switch (value) { + case ReaderOperation::OPEN_TO_FIRST_BLOCK: + return "open_to_first_block"; + case ReaderOperation::FULL_SCAN: + return "full_scan"; + case ReaderOperation::PREDICATE_SCAN: + return "predicate_scan"; + case ReaderOperation::LIMIT_1: + return "limit_1"; + case ReaderOperation::LIMIT_1000: + return "limit_1000"; + } + return "unknown"; +} + +} // namespace doris::parquet_benchmark diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp index 490a0bde204a74..c62d3566e152ec 100644 --- a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -109,25 +109,6 @@ TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversOperationsEncodingsAndSche } } -TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversFixedWidthRawFilterAxes) { - const auto scenarios = reader_scenarios(); - for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { - for (const int selectivity : {1, 10, 50, 90}) { - for (const auto projection : - {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { - EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { - return scenario.operation == ReaderOperation::PREDICATE_SCAN && - scenario.encoding == encoding && scenario.null_percent == 10 && - scenario.null_pattern == Pattern::ALTERNATING && - scenario.selectivity_percent == selectivity && - scenario.projection == projection && scenario.schema_width == 32 && - scenario.predicate_position == 0; - })) << "missing fixed-width raw filter axis combination"; - } - } - } -} - TEST(ParquetBenchmarkScenariosTest, SelectionPlanDistinguishesClusteredAndSparseRuns) { const auto clustered = make_selection_plan(1000, 10, Pattern::CLUSTERED); EXPECT_EQ(clustered.total_rows, 1000); From 3b3aed696b4d680e1b0a1196d962d445a2ce625b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 13:26:03 +0800 Subject: [PATCH 19/24] [improvement](be) Optimize projected fixed-width Parquet predicate filtering (#65934) ### What problem does this PR solve? Issue Number: N/A Related PR: #65921 Problem Summary: When a fixed-width Parquet predicate column was also projected, Doris materialized all selected predicate values and compacted the complete column after filtering. The native decoder had already produced the values needed by the raw predicate, so this added an avoidable materialize-and-compact pass. The overhead is visible in TPC-DS Q88. ### What is changed? Evaluate eligible predicates on decoded fixed-width values and append only matching values to the projected column in the same decoder pass. - Generalize the PLAIN-specific consumer and reader APIs into a fixed-width raw filter-and-project path. - Support PLAIN and BYTE_STREAM_SPLIT for identity-width INT32, INT64, FLOAT, and DOUBLE values. - Support DELTA_BINARY_PACKED for INT32 and INT64 values. - Prevalidate advertised chunk encodings before consuming definition levels. - Reject an unexpected unsupported late-page encoding instead of attempting a fallback after cursor progress. - Keep the existing dictionary-id filter and dictionary materialization path unchanged. The raw decoder cannot rewind after predicate evaluation. Therefore, when the predicate column is projected, survivors must be appended before the encoded values and definition-level cursor are consumed. Predicate-only columns retain their placeholder behavior. Nested columns, converted logical types, residual/delete predicates, unsupported expressions, and unsupported encodings continue to use the existing materializing fallback. ### Microbenchmark The reader microbenchmark from #65921 was run with a Release build, warm fixture cache, CPU 8, a one-second minimum time, and 10 repetitions. The matrix adds BYTE_STREAM_SPLIT and DELTA_BINARY_PACKED coverage with: - 10% alternating NULLs; - predicate-only and predicate-projected modes; - 1%, 10%, 50%, and 90% selectivity. All 16 combinations completed with the expected raw and selected row counts. Projected, 10% selectivity: | Encoding | CPU ns/raw row, master | CPU ns/raw row, PR | Improvement | CPU CV master / PR | |---|---:|---:|---:|---:| | BYTE_STREAM_SPLIT | 36.41 | 33.43 | 8.2% | 1.25% / 1.47% | | DELTA_BINARY_PACKED | 38.75 | 36.84 | 4.9% | 5.40% / 2.94% | The Delta baseline ran under sustained host load, so its result should be treated as directional. The original projected PLAIN results remain: | Selectivity | CPU ns/raw row, master | CPU ns/raw row, PR | Improvement | |---:|---:|---:|---:| | 1% | 32.80 | 30.80 | 6.1% | | 10% | 33.95 | 31.82 | 6.3% | | 50% | 37.76 | 35.05 | 7.2% | | 90% | 42.06 | 37.66 | 10.5% | ### Release note None ### Check List (For Author) - Test: Unit Test and Manual test - 17 targeted ASAN unit tests covering raw expression evaluation, nullable mapping, projected PLAIN/BSS/Delta scans, unsupported-type fallback, mixed-page safety, residual predicates, and benchmark matrix constraints - Release benchmark target linked successfully - 16-case BSS/Delta reader microbenchmark matrix - Before/after projected-filter microbenchmarks - clang-format 16, check-format, and `git diff --check` - clang-tidy was attempted but blocked by existing master/toolchain errors: unmatched `NOLINTEND` in `be/src/core/types.h` and missing `stddef.h` from the configured toolchain - Behavior changed: No. This extends the existing raw predicate optimization to equivalent fixed-width decoder output. - Does this need documentation: No (cherry picked from commit 187154675fec90fdfefdd93521d58a7722105fe1) --- .../parquet/parquet_benchmark_scenarios.h | 13 +++++++++++++ .../parquet_benchmark_scenarios_test.cpp | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/be/benchmark/parquet/parquet_benchmark_scenarios.h b/be/benchmark/parquet/parquet_benchmark_scenarios.h index d420d9ccdd552c..a01c955c6a2aed 100644 --- a/be/benchmark/parquet/parquet_benchmark_scenarios.h +++ b/be/benchmark/parquet/parquet_benchmark_scenarios.h @@ -126,6 +126,19 @@ inline std::vector reader_scenarios() { scenario.operation = ReaderOperation::PREDICATE_SCAN; add(scenario); } + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.encoding = encoding; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } for (const int width : {4, 32, 128, 512}) { for (const int predicate_position : {0, width - 1}) { auto scenario = baseline; diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp index c62d3566e152ec..490a0bde204a74 100644 --- a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -109,6 +109,25 @@ TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversOperationsEncodingsAndSche } } +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversFixedWidthRawFilterAxes) { + const auto scenarios = reader_scenarios(); + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == encoding && scenario.null_percent == 10 && + scenario.null_pattern == Pattern::ALTERNATING && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32 && + scenario.predicate_position == 0; + })) << "missing fixed-width raw filter axis combination"; + } + } + } +} + TEST(ParquetBenchmarkScenariosTest, SelectionPlanDistinguishesClusteredAndSparseRuns) { const auto clustered = make_selection_plan(1000, 10, Pattern::CLUSTERED); EXPECT_EQ(clustered.total_rows, 1000); From 9259acb1d4dd300f9f53c56f03f9568edffbacc4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 16:52:33 +0800 Subject: [PATCH 20/24] [improvement](file) Execute split residual predicates in TableReader (#65925) - Add a split-local localization result so `TableReader` knows which table predicates are enforced exactly by the current `FileReader`. - Evaluate only unlocalized predicates at the end of `TableReader::finalize_chunk`, after schema evolution, defaults, partition values, and virtual columns are materialized. - Make `FileScannerV2` skip row-level conjunct evaluation; it keeps scheduling and accounting responsibilities only. - Preserve predicate order by keeping unsafe/stateful predicates and every later predicate on the table-materialization path. - Apply the same table-level residual filtering to the Iceberg position-delete system-table reader. `FileScannerV2` previously reevaluated every original conjunct after `FileReader` had already enforced localized predicates. This duplicated expression work and made predicate execution ownership unclear. Localization also depends on each split's physical schema, so ownership cannot be cached at scanner or table-reader lifetime scope. The new contract recomputes ownership for every split: localized predicates belong to `FileReader`, while all remaining predicates belong to `TableReader` after final materialization. - ASAN BE UT build: `cmake --build be/ut_build_ASAN --target doris_be_test -j128` - 287 focused TableReader/ColumnMapper/Iceberg/FileScannerV2 tests passed. - 41 JNI/Hudi/Paimon reader tests passed. - `build-support/check-format.sh` passed with clang-format 16.0.6. - clang-tidy was attempted but could not analyze the tree because of existing toolchain/baseline errors (`stddef.h` not found and an unmatched `NOLINTEND` in `core/types.h`). (cherry picked from commit 27cb451229c7a42dd089c5248847828eb0536d02) --- be/src/exec/operator/scan_operator.cpp | 21 +- be/src/exec/operator/scan_operator.h | 13 +- be/src/exec/scan/file_scanner_v2.cpp | 145 ++++++-- be/src/exec/scan/file_scanner_v2.h | 33 +- be/src/exec/scan/scanner.cpp | 16 +- be/src/exec/scan/scanner.h | 8 + be/src/format_v2/column_mapper.cpp | 69 +++- be/src/format_v2/column_mapper.h | 15 +- be/src/format_v2/file_reader.h | 4 +- be/src/format_v2/table/hudi_reader.cpp | 22 ++ be/src/format_v2/table/hudi_reader.h | 2 + ...eberg_position_delete_sys_table_reader.cpp | 31 +- be/src/format_v2/table/paimon_reader.cpp | 22 ++ be/src/format_v2/table/paimon_reader.h | 2 + be/src/format_v2/table_reader.cpp | 143 +++++++- be/src/format_v2/table_reader.h | 128 +++++-- .../segment/adaptive_block_size_predictor.cpp | 7 +- .../segment/adaptive_block_size_predictor.h | 1 + be/test/exec/scan/file_scanner_v2_test.cpp | 35 +- .../scan/scanner_late_arrival_rf_test.cpp | 57 ++- ..._position_delete_sys_table_reader_test.cpp | 73 ++++ be/test/format_v2/column_mapper_test.cpp | 68 +++- be/test/format_v2/table/hudi_reader_test.cpp | 147 ++++++++ .../format_v2/table/iceberg_reader_test.cpp | 11 - .../format_v2/table/paimon_reader_test.cpp | 146 ++++++++ be/test/format_v2/table_reader_test.cpp | 340 +++++++++++++++++- .../adaptive_block_size_predictor_test.cpp | 12 + 27 files changed, 1427 insertions(+), 144 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index f945fa0a488810..f3b209ac2dd540 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -73,17 +73,34 @@ bool ScanLocalState::should_run_serial() const { return _parent->cast()._should_run_serial; } -Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, - int& arrived_rf_num) { +Status ScanLocalStateBase::update_late_arrival_runtime_filter( + RuntimeState* state, int applied_rf_num, int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts) { // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads LockGuard lock(_conjuncts_lock); + arrived_conjuncts.clear(); + size_t conjuncts_before = _conjuncts.size(); RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter(state, _parent->row_descriptor(), arrived_rf_num, _conjuncts)); + if (_conjuncts.size() > conjuncts_before) { + VExprContextSPtrs appended(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + _late_arrival_conjunct_batches.emplace_back(arrived_rf_num, std::move(appended)); + } if (state->enable_adjust_conjunct_order_by_cost()) { std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { return a->execute_cost() < b->execute_cost(); }); }; + for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) { + if (batch_arrived_rf_num <= applied_rf_num) { + continue; + } + for (const auto& conjunct : batch) { + VExprContextSPtr cloned; + RETURN_IF_ERROR(conjunct->clone(state, cloned)); + arrived_conjuncts.push_back(std::move(cloned)); + } + } return Status::OK(); } diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index fb7d5ec992c793..a9a1f3f90f1dd9 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include "common/status.h" #include "common/thread_safety_annotations.h" @@ -89,11 +91,13 @@ class ScanLocalStateBase : public PipelineXLocalState<> { [[nodiscard]] std::string get_name() { return _parent->get_name(); } - Status update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num); + uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } - Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); + Status update_late_arrival_runtime_filter(RuntimeState* state, int applied_rf_num, + int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts); - uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } + Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); protected: friend class ScannerContext; @@ -130,6 +134,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { AnnotatedMutex _conjuncts_lock; RuntimeFilterConsumerHelper _helper; + // Preserve append identity independently of the cost-sorted operator snapshot. Every scanner + // needs the exact RF contexts added since its own applied count. + std::vector> _late_arrival_conjunct_batches; // magic number as seed to generate hash value for condition cache uint64_t _condition_cache_digest = 0; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index e9d5b67dfa27f1..2ebcf65b2ce944 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -220,7 +221,9 @@ Status rewrite_slot_refs_to_global_index( #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) - : Scanner(state, profile), _table_reader(std::move(table_reader)) {} + : Scanner(state, profile), + _table_reader(std::move(table_reader)), + _scanner_profile(profile) {} Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -320,7 +323,9 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); + _initialize_scanner_residual_conjuncts(); auto* profile = _local_state->scanner_profile(); + _scanner_profile = profile; const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); _scanner_total_timer = hierarchy.scanner; _io_timer = hierarchy.io; @@ -354,6 +359,11 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1); + _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _refresh_scanner_residual_profile(); SCOPED_TIMER(_scanner_total_timer); SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); @@ -395,6 +405,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); + RETURN_IF_ERROR(_sync_table_reader_conjuncts()); if (!_has_prepared_split) { RETURN_IF_ERROR(_prepare_next_split(eof)); if (*eof) { @@ -444,18 +455,33 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } Status FileScannerV2::_filter_output_block(Block* block) { - return _contextualize_output_filter_status(Scanner::_filter_output_block(block), - _get_current_format_type()); -} - -Status FileScannerV2::_contextualize_output_filter_status(Status status, - TFileFormatType::type format_type) { - if (!status.ok() && format_type == TFileFormatType::FORMAT_ORC) { - // Error-preserving expressions cannot be reordered into the ORC reader and therefore run - // at the scanner boundary; keep their error context identical to ORC callback failures. + if (_scanner_residual_conjuncts.empty() || block->rows() == 0) { + return Status::OK(); + } + SCOPED_TIMER(_scanner_residual_filter_timer); + const size_t rows_before_filter = block->rows(); + auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns()); + if (!status.ok() && _params != nullptr && + _get_current_format_type() == TFileFormatType::FORMAT_ORC) { status.prepend("Orc row reader nextBatch failed. reason = "); } - return status; + RETURN_IF_ERROR(status); + const int64_t filtered_rows = cast_set(rows_before_filter - block->rows()); + _counter.num_rows_unselected += filtered_rows; + if (_scanner_residual_rows_filtered_counter != nullptr) { + COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows); + } + return Status::OK(); +} + +size_t FileScannerV2::_last_block_rows_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.rows : block.rows(); +} + +size_t FileScannerV2::_last_block_bytes_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes(); } Status FileScannerV2::_prepare_next_split(bool* eos) { @@ -541,6 +567,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = const_cast(_params), .io_ctx = _io_ctx, @@ -551,6 +578,9 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), })); + _table_reader_applied_rf_num = _applied_rf_num; + // RFs collected before TableReader initialization are already present in the full snapshot. + _late_arrival_rf_conjuncts.clear(); return Status::OK(); } @@ -595,15 +625,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values) { format::FileFormat current_split_format; RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), ¤t_split_format)); - VExprContextSPtrs conjuncts; - RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts)); VExprContextSPtrs partition_prune_conjuncts; if (_state->query_options().enable_runtime_filter_partition_prune) { RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts)); } RETURN_IF_ERROR(_table_reader->prepare_split({ .partition_values = std::move(partition_values), - .conjuncts = std::move(conjuncts), .partition_prune_conjuncts = std::move(partition_prune_conjuncts), // A metadata COUNT split may span scheduler turns. Do not enter that irreversible // synthetic-row path while a runtime filter can still arrive between batches. @@ -789,10 +816,15 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor } Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { + return _build_table_conjuncts(_conjuncts, conjuncts); +} + +Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const { DORIS_CHECK(conjuncts != nullptr); conjuncts->clear(); - conjuncts->reserve(_conjuncts.size()); - for (const auto& conjunct : _conjuncts) { + conjuncts->reserve(source.size()); + for (const auto& conjunct : source) { VExprSPtr root; RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); @@ -801,6 +833,68 @@ Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const return Status::OK(); } +size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) { + return conjunct_index; + } + } + return conjuncts.size(); +} + +void FileScannerV2::_initialize_scanner_residual_conjuncts() { + _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); + // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe + // predicate could run below Scanner before a stateful/error-preserving ordering barrier. + _scanner_residual_conjuncts.assign( + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count), + _conjuncts.end()); + _refresh_scanner_residual_profile(); +} + +void FileScannerV2::_refresh_scanner_residual_profile() { + if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) { + return; + } + std::ostringstream predicates; + predicates << "["; + for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size(); + ++conjunct_index) { + if (conjunct_index > 0) { + predicates << ", "; + } + predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string(); + } + predicates << "]"; + _scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str()); +} + +Status FileScannerV2::_sync_table_reader_conjuncts() { + if (_table_reader == nullptr) { + return Status::OK(); + } + if (_table_reader_applied_rf_num == _applied_rf_num) { + return Status::OK(); + } + VExprContextSPtrs appended; + RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended)); + const size_t owned_count = _scanner_residual_conjuncts.empty() + ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) + : 0; + // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting + // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. + RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); + _table_reader_owned_conjunct_count += owned_count; + _scanner_residual_conjuncts.insert( + _scanner_residual_conjuncts.end(), + _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), + _late_arrival_rf_conjuncts.end()); + _refresh_scanner_residual_profile(); + _late_arrival_rf_conjuncts.clear(); + _table_reader_applied_rf_num = _applied_rf_num; + return Status::OK(); +} + TFileFormatType::type FileScannerV2::_get_current_format_type() const { return get_range_format_type(*_params, _current_range); } @@ -925,17 +1019,19 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { if (!_should_run_adaptive_batch_size()) { return; } - COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(block.bytes())); - if (block.rows() == 0) { + const auto& stats = _table_reader->last_materialized_block_stats(); + const size_t rows = stats.has_materialized_input ? stats.rows : block.rows(); + const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes(); + COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(bytes)); + if (rows == 0) { return; } - // The sample is taken after TableReader has finalized file-local columns to table columns. - // This matches the memory shape seen by upstream operators and catches very wide nested - // columns, such as map/string payloads, after the first probe batch. + // Residual predicates run after wide table columns are materialized. Learn from that pre-filter + // shape so selective predicates cannot make the next reader batch dangerously large. if (!_block_size_predictor->has_history()) { COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1); } - _block_size_predictor->update(block); + _block_size_predictor->update(rows, bytes); } Status FileScannerV2::close(RuntimeState* state) { @@ -1127,9 +1223,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() { const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0; const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows; if (filtered_delta > 0) { - // File readers can evaluate localized conjuncts before a block reaches Scanner. Count - // those rows as scanner-level unselected rows so load statistics stay identical no matter - // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block(). + // FileReader and TableReader both report their owned predicate rows through the shared IO + // context. Preserve scanner-level load statistics without re-evaluating either predicate. _counter.num_rows_unselected += filtered_delta; _reported_predicate_filtered_rows = filtered_rows; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index fc217506eea0d3..c68cfb1db8b47c 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -88,15 +88,22 @@ class FileScannerV2 final : public Scanner { RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); - static Status TEST_contextualize_output_filter_status(Status status, - TFileFormatType::type format_type) { - return _contextualize_output_filter_status(std::move(status), format_type); - } static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, bool current_split_uses_metadata_count) { return _should_run_adaptive_batch_size(predictor_initialized, current_split_uses_metadata_count); } + void TEST_set_scanner_conjuncts(VExprContextSPtrs conjuncts) { + _conjuncts = std::move(conjuncts); + _initialize_scanner_residual_conjuncts(); + } + Status TEST_filter_output_block(Block* block) { return _filter_output_block(block); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + size_t TEST_scanner_residual_conjunct_count() const { + return _scanner_residual_conjuncts.size(); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -115,6 +122,8 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; Status _filter_output_block(Block* block) override; + size_t _last_block_rows_read(const Block& block) const override; + size_t _last_block_bytes_read(const Block& block) const override; void _collect_profile_before_close() override; bool _should_update_load_counters() const override; @@ -133,8 +142,6 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); static bool _should_skip_empty(const Status& status, bool stopped); - static Status _contextualize_output_filter_status(Status status, - TFileFormatType::type format_type); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -146,6 +153,12 @@ class FileScannerV2 final : public Scanner { Status _build_default_expr(const TFileScanSlotInfo& slot_info, VExprContextSPtr* ctx) const; static format::ColumnDefinition _build_table_column(const SlotDescriptor* slot_desc); Status _build_table_conjuncts(VExprContextSPtrs* conjuncts) const; + Status _build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const; + Status _sync_table_reader_conjuncts(); + static size_t _safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts); + void _initialize_scanner_residual_conjuncts(); + void _refresh_scanner_residual_profile(); static Status _to_file_format(TFileFormatType::type format_type, format::FileFormat* file_format); void _reset_adaptive_batch_size_state(); @@ -181,6 +194,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + size_t _table_reader_owned_conjunct_count = 0; + // Scanner owns one persistent context vector for the first unsafe conjunct and every later + // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. + VExprContextSPtrs _scanner_residual_conjuncts; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to @@ -214,6 +231,9 @@ class FileScannerV2 final : public Scanner { RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; + RuntimeProfile::Counter* _scanner_residual_filter_timer = nullptr; + RuntimeProfile::Counter* _scanner_residual_rows_filtered_counter = nullptr; + RuntimeProfile* _scanner_profile = nullptr; std::unique_ptr _block_size_predictor; int64_t _reported_predicate_filtered_rows = 0; int64_t _reported_condition_cache_hit_count = 0; @@ -223,6 +243,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; int64_t _reported_io_read_time = 0; + int _table_reader_applied_rf_num = 0; }; } // namespace doris diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index 3069438c764b90..c5a74b358e72da 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/config.h" #include "common/status.h" #include "core/block/column_with_type_and_name.h" @@ -145,8 +147,11 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) { DCHECK(block->rows() == 0); break; } - _num_rows_read += block->rows(); - _num_byte_read += block->allocated_bytes(); + // Some scanners apply owned predicates before returning the block. Account the + // materialized input, not only survivors, so the per-turn progress bound remains + // effective for highly selective predicates. + _num_rows_read += _last_block_rows_read(*block); + _num_byte_read += _last_block_bytes_read(*block); } // 2. Filter the output block finally. @@ -228,7 +233,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { } DCHECK(_applied_rf_num < _total_rf_num); int arrived_rf_num = 0; - RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter(_state, arrived_rf_num)); + VExprContextSPtrs arrived_conjuncts; + RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter( + _state, _applied_rf_num, arrived_rf_num, arrived_conjuncts)); if (arrived_rf_num == _applied_rf_num) { // No newly arrived runtime filters, just return; @@ -238,6 +245,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { // avoid conjunct destroy in used by storage layer _conjuncts.clear(); RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + _late_arrival_rf_conjuncts.insert(_late_arrival_rf_conjuncts.end(), + std::make_move_iterator(arrived_conjuncts.begin()), + std::make_move_iterator(arrived_conjuncts.end())); _applied_rf_num = arrived_rf_num; return Status::OK(); } diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index e90754db1c23d6..f12b6b2849f3ae 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -204,6 +204,11 @@ class Scanner { void update_block_avg_bytes(size_t block_avg_bytes) { _block_avg_bytes = block_avg_bytes; } protected: + virtual size_t _last_block_rows_read(const Block& block) const { return block.rows(); } + virtual size_t _last_block_bytes_read(const Block& block) const { + return block.allocated_bytes(); + } + RuntimeState* _state = nullptr; ScanLocalStateBase* _local_state = nullptr; @@ -231,6 +236,9 @@ class Scanner { // Cloned from _conjuncts of scan node. // It includes predicate in SQL and runtime filters. VExprContextSPtrs _conjuncts; + // Exact append-only RF delta for readers that preserve state across multiple splits. It must + // not be reconstructed by position from the cost-sorted full conjunct snapshot. + VExprContextSPtrs _late_arrival_rf_conjuncts; VExprContextSPtrs _projections; // Used in common subexpression elimination to compute intermediate results. std::vector _intermediate_projections; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 7457098a76feec..8ac3060bac48ab 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -399,6 +399,33 @@ std::string TableColumnMapperOptions::debug_string() const { return out.str(); } +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto primitive_type = table_type->get_primitive_type(); + if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { + return false; + } + const auto target_len = assert_cast(table_type.get())->len(); + if (target_len <= 0) { + return false; + } + if (mapping.file_type == nullptr) { + return true; + } + const auto file_type = remove_nullable(mapping.file_type); + DORIS_CHECK(file_type != nullptr); + int file_len = -1; + if (file_type->get_primitive_type() == TYPE_VARCHAR || + file_type->get_primitive_type() == TYPE_CHAR || + file_type->get_primitive_type() == TYPE_STRING) { + file_len = assert_cast(file_type.get())->len(); + } + return file_len < 0 || target_len < file_len; +} + std::string ColumnDefinition::debug_string() const { std::ostringstream out; out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) @@ -2156,7 +2183,7 @@ Status TableColumnMapper::_build_filter_entries(const FileScanRequest& file_requ Status TableColumnMapper::create_scan_request( const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, FilterLocalizationResult* localization_result) { // FileReader evaluates expressions against a file-local block. This mapper owns the // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); @@ -2192,7 +2219,8 @@ Status TableColumnMapper::create_scan_request( // 2. Build referenced predicate columns // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); - RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + RETURN_IF_ERROR( + localize_filters(table_filters, file_request, runtime_state, localization_result)); for (const auto& mapping : _hidden_mappings) { if (!mapping.file_local_id.has_value()) { continue; @@ -2206,9 +2234,9 @@ Status TableColumnMapper::create_scan_request( if (is_visible_output) { continue; } - // File-local filtering is an optimization; Scanner still evaluates the original - // table-level conjunct after TableReader returns. Only truly hidden mappings are absent - // from that scanner-visible block and may safely discard their payload here. + // A localized predicate is enforced exactly before TableReader materializes output. Only + // truly hidden mappings are absent from the final table block and may discard their + // payload after that file-local evaluation. if (std::ranges::any_of(file_request->predicate_columns, [local_id](const LocalColumnIndex& projection) { return projection.column_id() == local_id; @@ -2257,7 +2285,11 @@ ColumnMapping* TableColumnMapper::_find_filter_mapping(GlobalIndex global_index) Status TableColumnMapper::localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, + FilterLocalizationResult* localization_result) { + if (localization_result != nullptr) { + localization_result->localized_filters.assign(table_filters.size(), false); + } std::set localized_predicate_columns; FilterProjectionMap filter_projections; auto filter_mappings = _filter_visible_mappings(); @@ -2295,17 +2327,28 @@ Status TableColumnMapper::localize_filters(const std::vector& table // This keeps expression localization independent from filter iteration order. filter_mappings = _filter_visible_mappings(); const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); - for (const auto& table_filter : table_filters) { + for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { + const auto& table_filter = table_filters[filter_index]; if (table_filter.conjunct != nullptr && table_filter.conjunct->root() != nullptr) { const auto root = table_filter.conjunct->root(); const auto impl = root->get_impl(); const auto predicate = impl != nullptr ? impl : root; - if (!predicate->is_deterministic() || + if (!table_filter.can_localize || !predicate->is_deterministic() || !table_filter_has_only_local_entries(table_filter, _filter_entries)) { continue; } - // Scanner evaluates the original conjunct after final materialization. Only predicates - // whose result is stable across repeated execution may also run as a file-local copy. + if (runtime_state != nullptr && + runtime_state->query_options().truncate_char_or_varchar_columns && + std::ranges::any_of(table_filter.global_indices, [&](GlobalIndex global_index) { + const auto* mapping = _find_filter_mapping(global_index); + return mapping != nullptr && requires_char_or_varchar_truncation(*mapping); + })) { + // The table predicate observes the bounded value after finalize; evaluating it on + // a wider file string would change equality and range semantics. + continue; + } + // FileReader becomes the exact owner only for a stable predicate whose complete + // expression can be rewritten against this split's physical schema. RewriteContext rewrite_context {.runtime_state = runtime_state}; VExprSPtr rewrite_root; Status clone_status; @@ -2316,8 +2359,7 @@ Status TableColumnMapper::localize_filters(const std::vector& table // `element_at(MAP_VALUES(m)[1], 'age') > 30`. The current file-local rewrite only // understands top-level slots and struct-element paths rooted at top-level slots; // cloning such expressions can hit the generic TExpr complex-type limitation. - // Leave them above TableReader, where Scanner evaluates the original table-level - // conjunct after final materialization. + // Leave them for TableReader after final table-schema materialization. #ifndef NDEBUG return Status::InternalError( "Failed to clone table filter for file-local rewrite: {}, expr={}", @@ -2353,6 +2395,9 @@ Status TableColumnMapper::localize_filters(const std::vector& table auto localized_conjunct = VExprContext::create_shared(std::move(localized_root)); RETURN_IF_ERROR(rewrite_context.prepare_created_exprs(localized_conjunct.get())); file_request->conjuncts.push_back(std::move(localized_conjunct)); + if (localization_result != nullptr) { + localization_result->localized_filters[filter_index] = true; + } for (const auto global_index : table_filter.global_indices) { const auto* mapping = _find_filter_mapping(global_index); if (mapping != nullptr && mapping->file_local_id.has_value() && diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 0b42ac39a5cbea..1ee306a87741de 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -40,6 +40,13 @@ namespace doris::format { struct ColumnDefinition; struct TableFilter; +// Reports which table filters were fully rewritten into exact file-local predicates for the +// current split. The result is aligned with the TableFilter input vector and must not be reused for +// another split because schema evolution can change localization independently for every file. +struct FilterLocalizationResult { + std::vector localized_filters; +}; + enum class TableColumnMappingMode { // Match by ColumnDefinition::identifier TYPE_INT as field id. BY_FIELD_ID, @@ -164,6 +171,8 @@ struct TableColumnMapperOptions { std::string debug_string() const; }; +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping); + Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); @@ -195,7 +204,8 @@ class TableColumnMapper { virtual Status create_scan_request(const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); // Localize table-level filters to the file schema. // Trivial mappings can copy structured predicates directly. Type changes may be localized with @@ -203,7 +213,8 @@ class TableColumnMapper { // table-level finalize/filter fallback. virtual Status localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); void clear() { _mappings.clear(); _hidden_mappings.clear(); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 5f959c3e672dcd..1f90957064b254 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -76,7 +76,9 @@ struct FileScanRequest { std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; - // Row-level filters converted to file-local expressions from table-level predicates. + // Row-level filters converted to file-local expressions from table-level predicates. Readers + // must enforce these exactly on returned rows; metadata pruning alone does not transfer + // predicate ownership away from TableReader. VExprContextSPtrs conjuncts; // Delete predicates converted to file-local expressions. A TRUE result means that the row is // deleted, so readers must invert each result when building their keep filter. diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index d05cfe4b03e357..ef9c277254311f 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -125,6 +125,27 @@ void HudiHybridReader::set_batch_size(size_t batch_size) { } } +Status HudiHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& HudiHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { @@ -169,6 +190,7 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index d81e8c80e93ed3..42776422c4ad19 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -65,6 +65,8 @@ class HudiHybridReader final : public format::TableReader { Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; #ifdef BE_TEST void TEST_install_batch_size_children() { diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index b422243707e7db..16c7f829089e76 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -162,6 +162,12 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + // This synthetic reader has no physical schema where a predicate can be localized, so every + // split predicate must run after its system-table columns have been materialized. + RETURN_IF_ERROR(_prepare_all_conjuncts_as_remaining()); // The inner delete-file reader has distinct counters, so the outer preparation can safely // contain its cache miss/open work without re-entering the same RuntimeProfile timer. SCOPED_TIMER(_profile.total_timer); @@ -174,6 +180,7 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _reset_materialized_block_stats(); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(block->columns() == _projected_columns.size()); @@ -191,9 +198,19 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) return Status::OK(); } - size_t read_rows = 0; if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { - return _append_deletion_vector_block(block, &read_rows, eos); + size_t read_rows = 0; + RETURN_IF_ERROR(_append_deletion_vector_block(block, &read_rows, eos)); + if (read_rows > 0) { + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + } + if (read_rows == 0) { + // Yield after one deletion-vector batch so cancellation and Scanner row budgets are + // observed even when residual predicates reject every synthesized row. + block->clear_column_data(_projected_columns.size()); + } + return Status::OK(); } DORIS_CHECK(_position_reader != nullptr); @@ -207,8 +224,18 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) RETURN_IF_ERROR(_position_reader->get_block(&delete_block, &position_reader_eof)); const size_t delete_rows = delete_block.rows(); if (delete_rows > 0) { + size_t read_rows = 0; RETURN_IF_ERROR( _append_position_delete_block(block, delete_block, delete_rows, &read_rows)); + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + if (read_rows == 0) { + // A filtered materialized batch is still progress; return it to Scanner instead of + // consuming an unbounded number of position-delete batches in this call. + block->clear_column_data(_projected_columns.size()); + *eos = false; + return Status::OK(); + } *eos = false; return Status::OK(); } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 942fbb234269fc..3e409fee9da5ce 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -152,6 +152,27 @@ void PaimonHybridReader::set_batch_size(size_t batch_size) { } } +Status PaimonHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& PaimonHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); @@ -199,6 +220,7 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index ad29075fed1f08..6ee5e7d72f3405 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -71,6 +71,8 @@ class PaimonHybridReader final : public format::TableReader { Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; #ifdef BE_TEST static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 4beaf8c9ff5550..a073f01f237506 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -678,6 +678,8 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PrepareSplitTime", table_profile, 1); _profile.finalize_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "FinalizeBlockTime", table_profile, 1); + _profile.residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "ResidualFilterTime", table_profile, 1); _profile.create_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CreateReaderTime", table_profile, 1); _profile.pushdown_agg_timer = @@ -721,6 +723,9 @@ Status TableReader::init(TableReadOptions&& options) { _push_down_count_columns = options.push_down_count_columns; _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; + _table_reader_owned_conjunct_count = + options.table_reader_owned_conjunct_count.value_or(options.conjuncts.size()); + DORIS_CHECK_LE(_table_reader_owned_conjunct_count, options.conjuncts.size()); _projected_columns = std::move(options.projected_columns); if (supports_iceberg_scan_semantics_v1(_scan_params)) { for (auto& projected_column : _projected_columns) { @@ -734,7 +739,64 @@ Status TableReader::init(TableReadOptions&& options) { } _system_properties = create_system_properties(_scan_params); _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); + return _replace_conjuncts(options.conjuncts); +} + +Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { + DORIS_CHECK(source != nullptr); + DORIS_CHECK(source->root() != nullptr); + DORIS_CHECK(prepared != nullptr); + VExprSPtr root; + RETURN_IF_ERROR(clone_table_expr_tree(source->root(), &root)); + auto conjunct = VExprContext::create_shared(std::move(root)); + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + *prepared = std::move(conjunct); + return Status::OK(); +} + +Status TableReader::_replace_conjuncts(const VExprContextSPtrs& conjuncts) { + VExprContextSPtrs prepared; + prepared.reserve(conjuncts.size()); + for (const auto& source : conjuncts) { + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + prepared.push_back(std::move(conjunct)); + } + _conjuncts = std::move(prepared); + return Status::OK(); +} + +Status TableReader::append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count) { + DORIS_CHECK(!_appended_table_reader_owned_conjunct_count.has_value()); + _appended_table_reader_owned_conjunct_count = table_reader_owned_conjunct_count; + auto status = append_conjuncts(conjuncts); + _appended_table_reader_owned_conjunct_count.reset(); + return status; +} + +Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + DORIS_CHECK_LE(owned_count, conjuncts.size()); + // Once Scanner owns a suffix, later predicates cannot be inserted into the TableReader-owned + // prefix without reordering them ahead of that stateful/error-preserving barrier. + DORIS_CHECK(owned_count == 0 || _table_reader_owned_conjunct_count == _conjuncts.size()); + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + const auto& source = conjuncts[conjunct_index]; + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + _conjuncts.push_back(conjunct); + if (conjunct_index < owned_count) { + ++_table_reader_owned_conjunct_count; + } + if (_current_task != nullptr && conjunct_index < owned_count) { + // The active reader has already fixed its localized predicate set. Appended runtime + // filters must remain residual until the next split rebuilds its FileScanRequest. + _remaining_conjuncts.push_back(std::move(conjunct)); + } + } return Status::OK(); } @@ -742,20 +804,27 @@ Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); _constant_pruning_safe_filter_count = 0; bool in_safe_prefix = true; - for (const auto& conjunct : _conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { + const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); // `_table_filters` omits expressions without slot references, but such an expression still // occupies a position in the row-level conjunct order. Record how many localized filters // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. Unsafe predicates remain solely on - // Scanner's original row-level path because localizing a clone would execute their state - // twice with independent state. - if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { + // slotless non-deterministic/error-preserving barrier. An unsafe predicate is either kept + // on TableReader's post-materialization path by a standalone caller or carried only for + // analysis when FileScannerV2 owns the ordered suffix. + if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { in_safe_prefix = false; } + const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); + for (size_t filter_index = filters_before; filter_index < _table_filters.size(); + ++filter_index) { + _table_filters[filter_index].source_conjunct_index = conjunct_index; + _table_filters[filter_index].can_localize = in_safe_prefix; + } if (in_safe_prefix) { _constant_pruning_safe_filter_count = _table_filters.size(); } @@ -763,6 +832,59 @@ Status TableReader::_build_table_filters_from_conjuncts() { return Status::OK(); } +Status TableReader::_prepare_all_conjuncts_as_remaining() { + // Expression contexts carry mutable state (for example sequence/stateful functions). Select + // from the TableReader-owned contexts instead of reopening clones for every split. + _remaining_conjuncts.assign( + _conjuncts.begin(), + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count)); + return Status::OK(); +} + +Status TableReader::_prepare_remaining_conjuncts( + const FilterLocalizationResult& localization_result) { + DORIS_CHECK(localization_result.localized_filters.size() == _table_filters.size()); + std::vector localized_conjuncts(_conjuncts.size(), false); + for (size_t filter_index = 0; filter_index < _table_filters.size(); ++filter_index) { + if (!localization_result.localized_filters[filter_index]) { + continue; + } + const size_t source_index = _table_filters[filter_index].source_conjunct_index; + DORIS_CHECK(source_index < localized_conjuncts.size()); + localized_conjuncts[source_index] = true; + } + + _remaining_conjuncts.clear(); + for (size_t conjunct_index = 0; conjunct_index < _table_reader_owned_conjunct_count; + ++conjunct_index) { + if (localized_conjuncts[conjunct_index]) { + continue; + } + _remaining_conjuncts.push_back(_conjuncts[conjunct_index]); + } + return Status::OK(); +} + +Status TableReader::_filter_remaining_conjuncts(Block* block, size_t* rows) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + if (*rows == 0 || _remaining_conjuncts.empty()) { + return Status::OK(); + } + SCOPED_TIMER(_profile.residual_filter_timer); + const size_t rows_before_filter = *rows; + auto status = VExprContext::filter_block(_remaining_conjuncts, block, block->columns()); + if (!status.ok() && _format == FileFormat::ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + *rows = block->columns() == 0 ? rows_before_filter : block->rows(); + if (_io_ctx != nullptr) { + _io_ctx->predicate_filtered_rows += rows_before_filter - *rows; + } + return Status::OK(); +} + Status TableReader::_open_local_filter_exprs(const FileScanRequest& file_request) { RowDescriptor row_desc; for (const auto& conjunct : file_request.conjuncts) { @@ -1003,6 +1125,9 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; + // Predicate localization belongs to the physical schema of one split. Clear the previous + // ownership before any early return so a pruned or failed split cannot leak it to the next one. + _remaining_conjuncts.clear(); _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; _condition_cache_digest_covers_current_split = options.condition_cache_digest.has_value(); if (options.condition_cache_digest.has_value()) { @@ -1016,7 +1141,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _condition_cache_digest = _initial_condition_cache_digest; } if (options.conjuncts.has_value()) { - _conjuncts = *options.conjuncts; + RETURN_IF_ERROR(_replace_conjuncts(*options.conjuncts)); } // Update to current split format to handle ORC/PARQUET files in one table. _format = options.current_split_format; @@ -1086,7 +1211,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is // skipped, a later predicate could prune the split before the unsafe one reaches its // normal row-level evaluation point. - if (!_is_safe_to_pre_execute(conjunct)) { + if (!is_safe_to_pre_execute(conjunct)) { break; } std::set global_indices; @@ -1122,7 +1247,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& can_filter_all); } -bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { +bool TableReader::is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); const auto root = conjunct->root(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 7adbb3b0a5225c..47d442f7fc17f6 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -78,10 +78,14 @@ namespace doris::format { using DeleteRows = std::vector; // Row-level predicates on table/global schema. They are rewritten to file-local expressions when -// possible, and remain the source of row-level filtering after localization. +// possible; otherwise TableReader evaluates them after final table-schema materialization. struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; + size_t source_conjunct_index = 0; + // False after the first unsafe source conjunct so file-local execution cannot reorder a later + // predicate ahead of stateful or error-preserving table semantics. + bool can_localize = true; }; struct ScanTask { @@ -112,6 +116,7 @@ struct ReadProfile { RuntimeProfile::Counter* exec_timer = nullptr; RuntimeProfile::Counter* prepare_split_timer = nullptr; RuntimeProfile::Counter* finalize_timer = nullptr; + RuntimeProfile::Counter* residual_filter_timer = nullptr; RuntimeProfile::Counter* create_reader_timer = nullptr; RuntimeProfile::Counter* pushdown_agg_timer = nullptr; RuntimeProfile::Counter* open_reader_timer = nullptr; @@ -128,12 +133,23 @@ struct ReadProfile { RuntimeProfile::Counter* file_reader_close_timer = nullptr; }; +struct MaterializedBlockStats { + bool has_materialized_input = false; + size_t rows = 0; + size_t bytes = 0; + size_t allocated_bytes = 0; +}; + struct TableReadOptions { // Columns need to be read from file and output by table reader. They are all in table/global // schema semantics. const std::vector projected_columns; // All complex conjuncts from scan operator const VExprContextSPtrs conjuncts; + // Number of leading conjuncts whose row-level execution is owned by TableReader/FileReader. + // FileScannerV2 still passes the complete ordered list so mapping, pruning guards, aggregate + // eligibility, and condition-cache analysis see the exact query semantics. nullopt means all. + const std::optional table_reader_owned_conjunct_count = std::nullopt; // File format of the underlying data files, needed for reader initialization and reader-level // filter pushdown. const FileFormat format; @@ -206,6 +222,10 @@ class TableReader { #ifdef BE_TEST size_t TEST_batch_size() const { return _batch_size; } + size_t TEST_conjunct_count() const { return _conjuncts.size(); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } bool TEST_current_data_file_is_immutable() const { DORIS_CHECK(_current_task != nullptr); DORIS_CHECK(_current_task->data_file != nullptr); @@ -226,6 +246,25 @@ class TableReader { return _current_split_uses_metadata_count; } + // Runtime filters that arrive after a split has opened cannot be pushed into that file reader. + // Keep their expression contexts in TableReader and evaluate them as residual predicates for + // the active reader; later splits can localize them normally. + virtual Status append_conjuncts(const VExprContextSPtrs& conjuncts); + + // Append a full ordered snapshot delta while marking only its leading prefix as owned by + // TableReader/FileReader. This non-virtual wrapper preserves the long-standing virtual API and + // carries the ownership boundary through hybrid readers to their children. + Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count); + + // Shared safety classification for deciding which ordered conjunct prefix may execute below + // Scanner without changing stateful or error-preserving semantics. + static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); + + virtual const MaterializedBlockStats& last_materialized_block_stats() const { + return _last_materialized_block_stats; + } + // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. @@ -245,6 +284,7 @@ class TableReader { _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; _current_split_pruned = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -254,6 +294,7 @@ class TableReader { virtual Status get_block(Block* block, bool* eos) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _last_materialized_block_stats = {}; DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -326,7 +367,7 @@ class TableReader { RETURN_IF_ERROR(_check_file_block_columns("after file reader get_block", current_rows)); #endif DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); - RETURN_IF_ERROR(finalize_chunk(block, current_rows)); + RETURN_IF_ERROR(finalize_chunk(block, ¤t_rows)); #ifndef NDEBUG RETURN_IF_ERROR( _check_table_block_columns("after finalize_chunk", block, current_rows)); @@ -335,6 +376,13 @@ class TableReader { _current_reader_reached_eof = !stopped_during_read; RETURN_IF_ERROR(close_current_reader()); } + if (current_rows == 0) { + // One materialized batch is one Scanner progress unit even when residual + // predicates reject every row. Returning here preserves row-budget and + // cancellation checks in Scanner::get_block(). + block->clear_column_data(_projected_columns.size()); + return Status::OK(); + } return Status::OK(); } } @@ -352,6 +400,7 @@ class TableReader { _remaining_table_level_count = -1; _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -437,14 +486,17 @@ class TableReader { // reader with the request. File scan request carries row-level expression filters and // file-level pruning hints. Only expression filters decide returned rows. auto file_request = std::make_shared(); + FilterLocalizationResult localization_result; RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( - _table_filters, _projected_columns, file_request.get(), _runtime_state)); + _table_filters, _projected_columns, file_request.get(), _runtime_state, + &localization_result)); bool constant_filter_pruned_split = false; RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); if (constant_filter_pruned_split) { RETURN_IF_ERROR(close_current_reader()); return Status::OK(); } + RETURN_IF_ERROR(_prepare_remaining_conjuncts(localization_result)); // COUNT(*) has no semantic column argument, but Nereids retains a minimum-width scan slot // so the scan node still has an output tuple. Record only the current non-predicate file // columns before table-format hooks add row-position or equality-delete dependencies. This @@ -520,9 +572,13 @@ class TableReader { } Status _build_table_filters_from_conjuncts(); + Status _replace_conjuncts(const VExprContextSPtrs& conjuncts); + Status _prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared); + Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); + Status _prepare_all_conjuncts_as_remaining(); + Status _filter_remaining_conjuncts(Block* block, size_t* rows); Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all); - static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct); Status _build_partition_prune_block(Block* block) const; Status _open_local_filter_exprs(const FileScanRequest& file_request); Status _init_reader_condition_cache(const FileScanRequest& file_request); @@ -541,7 +597,7 @@ class TableReader { if (table_filter.conjunct == nullptr) { continue; } - DORIS_CHECK(_is_safe_to_pre_execute(table_filter.conjunct)); + DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); // RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the // row-level filter path through execute_filter(). Constant split pruning uses // VExprContext::execute() on a one-row synthetic block, so runtime filters must not be @@ -758,6 +814,7 @@ class TableReader { } _table_filters.clear(); _constant_pruning_safe_filter_count = 0; + _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); @@ -773,15 +830,28 @@ class TableReader { } } + void _reset_materialized_block_stats() { _last_materialized_block_stats = {}; } + + void _record_materialized_block_stats(const Block& block, size_t rows) { + _last_materialized_block_stats = { + .has_materialized_input = true, + .rows = rows, + .bytes = block.bytes(), + .allocated_bytes = block.allocated_bytes(), + }; + } + // Finalize file-local block to table/global schema block. - Status finalize_chunk(Block* block, const size_t rows) { + Status finalize_chunk(Block* block, size_t* rows) { + DORIS_CHECK(rows != nullptr); SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; const auto& mappings = _data_reader.column_mapper->mappings(); for (const auto& mapping : mappings) { ColumnPtr column; - RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column, idx + 1 == mappings.size())); + RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, + *rows, &column, + idx + 1 == mappings.size())); block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } @@ -789,7 +859,12 @@ class TableReader { // Enforce CHAR/VARCHAR length declared by the table schema after all file-to-table // materialization has finished. RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block)); - return Status::OK(); + // Preserve the cost of materialization before residual predicates shrink the block. The + // scanner uses this snapshot for bounded progress and adaptive batch sizing. + _record_materialized_block_stats(*block, *rows); + // Predicate ownership is split-local: only predicates not acknowledged as exact by this + // split's FileScanRequest run here, after virtual/default/schema-evolution values exist. + return _filter_remaining_conjuncts(block, rows); } // Materialize virtual columns in the table block, such as Iceberg _row_id and @@ -953,31 +1028,7 @@ class TableReader { // - table VARCHAR(10), file STRING: truncate to 10 because STRING has no declared bound; // - table STRING, any file type: no truncation because the target has no bound. static bool _should_truncate_char_or_varchar_column(const ColumnMapping& mapping) { - if (mapping.table_type == nullptr) { - return false; - } - const auto table_type = remove_nullable(mapping.table_type); - const auto primitive_type = table_type->get_primitive_type(); - if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { - return false; - } - const auto target_len = assert_cast(table_type.get())->len(); - if (target_len <= 0) { - return false; - } - if (mapping.file_type == nullptr) { - return true; - } - const auto file_type = remove_nullable(mapping.file_type); - DORIS_CHECK(file_type != nullptr); - int file_len = -1; - if (file_type->get_primitive_type() == TYPE_VARCHAR || - file_type->get_primitive_type() == TYPE_CHAR || - file_type->get_primitive_type() == TYPE_STRING) { - file_len = assert_cast(file_type.get())->len(); - } - - return file_len < 0 || target_len < file_len; + return requires_char_or_varchar_truncation(mapping); } // Truncate a materialized CHAR/VARCHAR column in place by reusing the vectorized substring @@ -1074,9 +1125,8 @@ class TableReader { if (!_all_runtime_filters_applied_for_split) { return false; } - // Scanner owns the original conjunct list and evaluates it after TableReader finalizes - // rows. Even a slotless conjunct that cannot become a TableFilter must see every source - // row before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. + // Even a slotless conjunct that cannot become a TableFilter must see every source row + // before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. if (!_conjuncts.empty()) { return false; } @@ -1748,6 +1798,10 @@ class TableReader { // intentionally absent from that vector but must still act as ordering barriers. size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; + size_t _table_reader_owned_conjunct_count = 0; + std::optional _appended_table_reader_owned_conjunct_count; + VExprContextSPtrs _remaining_conjuncts; + MaterializedBlockStats _last_materialized_block_stats; ReadProfile _profile; // Parsed from row-position based delete files, including position delete and deletion vector. DeleteRows* _delete_rows = nullptr; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp b/be/src/storage/segment/adaptive_block_size_predictor.cpp index d8cc700f579853..7a5ad573a2ea6c 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.cpp +++ b/be/src/storage/segment/adaptive_block_size_predictor.cpp @@ -32,11 +32,14 @@ AdaptiveBlockSizePredictor::AdaptiveBlockSizePredictor(size_t preferred_block_si _metadata_hint_bytes_per_row(metadata_hint_bytes_per_row) {} void AdaptiveBlockSizePredictor::update(const Block& block) { - size_t rows = block.rows(); + update(block.rows(), block.bytes()); +} + +void AdaptiveBlockSizePredictor::update(size_t rows, size_t bytes) { if (rows == 0) { return; } - double cur = static_cast(block.bytes()) / static_cast(rows); + double cur = static_cast(bytes) / static_cast(rows); if (!_has_history) { _bytes_per_row = cur; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h b/be/src/storage/segment/adaptive_block_size_predictor.h index e03f18c2a536d2..f327fec8517f63 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.h +++ b/be/src/storage/segment/adaptive_block_size_predictor.h @@ -60,6 +60,7 @@ class AdaptiveBlockSizePredictor { // Update EWMA estimates from a completed batch. Must be called only when block.rows() > 0 // and the batch returned Status::OK(). void update(const Block& block); + void update(size_t rows, size_t bytes); // Predict how many rows the next batch should read. // Never exceeds |block_size_rows|; never returns less than 1. diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 97591acf746159..98f086a4b12ec2 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -658,18 +658,6 @@ TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); } -TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) { - auto status = FileScannerV2::TEST_contextualize_output_filter_status( - Status::InvalidArgument("synthetic row filter failure"), TFileFormatType::FORMAT_ORC); - EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; - EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; - - status = FileScannerV2::TEST_contextualize_output_filter_status( - Status::InvalidArgument("synthetic row filter failure"), - TFileFormatType::FORMAT_PARQUET); - EXPECT_EQ(status.to_string().find("nextBatch failed"), std::string::npos) << status; -} - // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. @@ -779,4 +767,27 @@ TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); } +TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + VExprContextSPtrs conjuncts { + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1), + runtime_filter_context(std::move(unsafe_predicate), 2), + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3), + }; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("file_scanner_v2"); + FileScannerV2 scanner(&state, &profile, nullptr); + scanner.TEST_set_scanner_conjuncts(std::move(conjuncts)); + + EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 1); + EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 2); + const auto* residual_predicates = profile.get_info_string("ScannerResidualPredicates"); + ASSERT_NE(residual_predicates, nullptr); + EXPECT_FALSE(residual_predicates->empty()); + EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; +} + } // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 0d31b69495187e..b70f1cf9ea0a18 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -65,6 +65,23 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class HighCostPredicate final : public VExpr { +public: + HighCostPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + double execute_cost() const override { return 100.0; } + +private: + const std::string _expr_name = "high_cost_stateful_predicate"; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -83,6 +100,7 @@ class ScannerLateArrivalRfTest : public RuntimeFilterTest { // the counter advances after RFs arrive and that the second call short-circuits // via the fast path at the top of the function. TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { + _runtime_states[0]->_query_options.__set_enable_adjust_conjunct_order_by_cost(true); std::vector rf_descs = { TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build(), TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build()}; @@ -104,26 +122,55 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { auto local_state = std::make_shared(_runtime_states[0].get(), op.get()); + auto initial_conjunct = VExprContext::create_shared(std::make_shared()); + ASSERT_TRUE(initial_conjunct->prepare(_runtime_states[0].get(), row_desc).ok()); + ASSERT_TRUE(initial_conjunct->open(_runtime_states[0].get()).ok()); + local_state->_conjuncts.push_back(initial_conjunct); + std::vector> rf_dependencies; ASSERT_TRUE(local_state->_helper.init(_runtime_states[0].get(), true, 0, 0, rf_dependencies, "") .ok()); + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + local_state->_helper._consumers[0]->signal(producer.get()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(_runtime_states[0].get(), local_state->_conjuncts, + row_desc) + .ok()); + ASSERT_EQ(local_state->_conjuncts.size(), 2); + auto scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), -1 /*limit*/, &_profile); - ASSERT_TRUE(scanner->init(_runtime_states[0].get(), {}).ok()); + ASSERT_TRUE(scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); + auto second_scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), + -1 /*limit*/, &_profile); + ASSERT_TRUE(second_scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); ASSERT_EQ(scanner->_total_rf_num, 2); ASSERT_EQ(scanner->_applied_rf_num, 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - local_state->_helper._consumers[0]->signal(producer.get()); local_state->_helper._consumers[1]->signal(producer.get()); // First call after both RFs arrived: counter must advance to total. Before // the fix this stayed at 0 because the assignment was missing. ASSERT_TRUE(scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(scanner->_applied_rf_num, 2); + ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); + for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + } + ASSERT_EQ(scanner->_conjuncts.size(), 3); + EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); + + // The first scanner consumes the shared helper's expression, so another scanner can only get + // the exact delta from the local state's append-only RF batch history. + ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); + ASSERT_EQ(second_scanner->_applied_rf_num, 2); + ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); + EXPECT_NE(dynamic_cast( + second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), + nullptr); // Second call: must hit the fast-path early return without re-cloning. // We clear `_conjuncts` and verify the function does NOT repopulate them; diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index bff48b9a6941d3..e2ddebb983e899 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -29,10 +29,12 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -55,6 +57,31 @@ class ProfileTrackingReader final : public GenericReader { void _collect_profile_before_close() override { ++collect_calls; } }; +class RejectAllRowsPredicate final : public VExpr { +public: + RejectAllRowsPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 0); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(); + return Status::OK(); + } + +private: + const std::string _name = "RejectAllRowsPredicate"; +}; + SlotDescriptor* make_slot(ObjectPool* pool, int id, std::string name, DataTypePtr type) { TSlotDescriptor slot_desc; slot_desc.__set_id(id); @@ -605,4 +632,50 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVect EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, + AllFilteredDeletionVectorYieldsBeforeObservingCancellation) { + ObjectPool pool; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector file_slot_descs { + make_slot(&pool, 0, "pos", nullable_int64), + }; + + auto conjunct = VExprContext::create_shared(std::make_shared()); + RowDescriptor row_desc; + ASSERT_TRUE(conjunct->prepare(&state, row_desc).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._io_ctx = std::make_shared(); + reader._file_slot_descs = &file_slot_descs; + reader._projected_columns.resize(file_slot_descs.size()); + reader._remaining_conjuncts = {conjunct}; + reader._has_split = true; + reader._delete_file_kind = + format::iceberg::IcebergPositionDeleteSysTableV2Reader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(file_slot_descs); + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_FALSE(eof); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); + + reader._io_ctx->should_stop = true; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); +} + } // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 4bb7b920231a68..6bfc06139e4850 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2315,6 +2315,39 @@ TEST(ColumnMapperLocalizeFiltersTest, VisibleLocalFilterAddsPredicateColumnAndCo EXPECT_TRUE(localized_slot->data_type()->equals(*int_type)); } +TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { + const auto int_type = i32(); + auto table_column = name_col("id", int_type); + const std::vector table_schema = {table_column}; + TableFilter filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, int_type, "id"), 1)), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper local_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(local_mapper.create_mapping(table_schema, {}, {name_col("id", int_type, 7)}).ok()); + FileScanRequest local_request; + FilterLocalizationResult local_result; + ASSERT_TRUE(local_mapper + .create_scan_request({filter}, table_schema, &local_request, nullptr, + &local_result) + .ok()); + ASSERT_EQ(local_result.localized_filters.size(), 1); + EXPECT_TRUE(local_result.localized_filters[0]); + ASSERT_EQ(local_request.conjuncts.size(), 1); + + TableColumnMapper missing_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(missing_mapper.create_mapping(table_schema, {}, {}).ok()); + FileScanRequest missing_request; + FilterLocalizationResult missing_result; + ASSERT_TRUE(missing_mapper + .create_scan_request({filter}, table_schema, &missing_request, nullptr, + &missing_result) + .ok()); + ASSERT_EQ(missing_result.localized_filters.size(), 1); + EXPECT_FALSE(missing_result.localized_filters[0]); + EXPECT_TRUE(missing_request.conjuncts.empty()); +} + TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { const auto binary_type = varbinary(); const auto table_column = name_col("partition_key", binary_type); @@ -2340,6 +2373,35 @@ TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { EXPECT_TRUE(request.conjuncts.empty()); } +TEST(ColumnMapperLocalizeFiltersTest, VarcharWidthTruncationFilterStaysAboveFileReader) { + const auto table_type = std::make_shared(3, TYPE_VARCHAR); + const auto file_type = std::make_shared(10, TYPE_VARCHAR); + const auto table_column = name_col("value", table_type); + const auto file_column = name_col("value", file_type, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(binary_predicate( + TExprOpcode::EQ, table_slot(0, 0, table_type, "value"), + literal(table_type, Field::create_field("abc")))), + .global_indices = {GlobalIndex(0)}}; + TQueryOptions query_options; + query_options.__set_truncate_char_or_varchar_columns(true); + RuntimeState state {query_options, TQueryGlobals()}; + FileScanRequest request; + FilterLocalizationResult localization_result; + + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, + &localization_result) + .ok()); + ASSERT_EQ(localization_result.localized_filters.size(), 1); + EXPECT_FALSE(localization_result.localized_filters[0]); + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); +} + TEST(ColumnMapperLocalizeFiltersTest, NestedVarbinaryFilterStaysAboveFileReader) { const auto table_column = struct_name_col( "payload", {name_col("id", i32()), name_col("binary_value", varbinary())}); @@ -2490,7 +2552,7 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } -TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsPayloadForScannerBoundary) { +TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsOutputPayload) { const auto int_type = i32(); auto quantity = name_col("ss_quantity", int_type); auto tax = name_col("ss_ext_tax", int_type); @@ -2515,8 +2577,8 @@ TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsPayloadForScannerB EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.non_predicate_columns.size(), 1); EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); - // The scanner evaluates its table-level conjuncts after TableReader returns, so a visible - // predicate slot cannot be replaced with a default-valued placeholder at the file boundary. + // A visible predicate slot is still part of the table output and cannot be replaced with a + // default-valued placeholder after file-local filtering. EXPECT_TRUE(request.predicate_only_columns.empty()); } diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 6f9bea89ffe943..b8dd97e57d585f 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -37,6 +37,9 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format_v2/column_data.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -141,6 +144,54 @@ class SlowInitTableReader final : public TableReader { } }; +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -262,6 +313,102 @@ TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 123); } +TEST(HudiHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(HudiHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("hudi_scanner_stateful_residual"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + run_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 782463e335a743..c42d68933f2014 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -1146,11 +1146,6 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { return ctx; } -void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { - const auto status = VExprContext::filter_block(conjuncts, block, block->columns()); - ASSERT_TRUE(status.ok()) << status; -} - TEST(IcebergV2ReaderTest, IcebergVirtualColumnsUseRowLineageMetadata) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_test"; @@ -1389,9 +1384,6 @@ TEST(IcebergV2ReaderTest, IcebergRowIdPredicateFiltersAfterRowLineageMaterializa bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1443,9 +1435,6 @@ TEST(IcebergV2ReaderTest, IcebergLastUpdatedSequencePredicateFiltersAfterMateria bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 3c85eb4632f86c..2434d748dfa109 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -45,6 +45,9 @@ #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/common/endian.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format/format_common.h" #include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" @@ -79,6 +82,54 @@ class SlowInitTableReader final : public TableReader { } }; +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -694,6 +745,101 @@ TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 321); } +TEST(PaimonHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(PaimonHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("paimon_scanner_stateful_residual"); + auto scan_params = make_local_parquet_scan_params(); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + run_split(FileFormat::JNI, make_paimon_jni_range()); + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 48b38d78e19662..d92bbf2b0ac927 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -297,6 +297,37 @@ class NonDeterministicPartitionPredicate final : public VExpr { const std::string _expr_name = "NonDeterministicPartitionPredicate"; }; +class StatefulSequencePredicate final : public VExpr { +public: + explicit StatefulSequencePredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(_observed_invocations != nullptr); + _observed_invocations->push_back(_invocation++); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_observed_invocations); + return Status::OK(); + } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulSequencePredicate"; +}; + class NullableArrayBigintDefaultExpr final : public VExpr { public: explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) @@ -534,6 +565,23 @@ void write_parquet_file(const std::string& file_path, int32_t id, const std::str builder.build())); } +void write_single_int_parquet_file(const std::string& file_path, const std::string& column_name, + int32_t value) { + auto schema = arrow::schema({arrow::field(column_name, arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + void write_struct_parquet_file(const std::string& file_path, int32_t id) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); arrow::StructBuilder builder( @@ -1055,6 +1103,8 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + int batch_count = 1; + int get_block_count = 0; std::shared_ptr last_request; std::optional last_aggregate_request; std::shared_ptr condition_cache_ctx; @@ -1093,7 +1143,7 @@ class FakeFileReader final : public FileReader { RETURN_IF_ERROR(FileReader::open(std::move(request))); _state->last_request = _request; ++_state->open_count; - _returned_batch = false; + _returned_batches = 0; return Status::OK(); } @@ -1102,7 +1152,8 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); DORIS_CHECK(_request != nullptr); - if (_returned_batch) { + ++_state->get_block_count; + if (_returned_batches >= _state->batch_count) { *rows = 0; *eof = true; return Status::OK(); @@ -1149,9 +1200,9 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; } - _returned_batch = true; + ++_returned_batches; *rows = 2; - *eof = _state->eof_with_first_batch; + *eof = _state->eof_with_first_batch && _returned_batches >= _state->batch_count; if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit && _state->condition_cache_ctx->filter_result != nullptr && !_state->condition_cache_ctx->filter_result->empty()) { @@ -1207,7 +1258,7 @@ class FakeFileReader final : public FileReader { private: std::vector _schema; std::shared_ptr _state; - bool _returned_batch = false; + int _returned_batches = 0; }; class FakeTableReader final : public TableReader { @@ -1381,13 +1432,54 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); EXPECT_FALSE(eos); + // The file was still opened, proving constant pruning did not jump over the unsafe predicate; + // the predicate is evaluated only after the resulting table row is materialized. EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_TRUE(predicate_executed); ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { +TEST(TableReaderTest, ScannerOwnedUnsafePredicateIsPassedButNotExecutedByTableReader) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); std::vector projected_columns; @@ -1404,6 +1496,7 @@ TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .table_reader_owned_conjunct_count = 0, .format = FileFormat::PARQUET, .scan_params = nullptr, .io_ctx = nullptr, @@ -1418,9 +1511,178 @@ TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_EQ(reader.TEST_conjunct_count(), 1); + EXPECT_EQ(reader.TEST_table_reader_owned_conjunct_count(), 0); EXPECT_FALSE(predicate_executed); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualExpressionStateSurvivesAcrossSplits) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + std::vector observed_invocations; + auto stateful_predicate = std::make_shared(&observed_invocations); + stateful_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, stateful_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + for (int split_index = 0; split_index < 2; ++split_index) { + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(observed_invocations, std::vector({0, 1})); +} + +TEST(TableReaderTest, AllFilteredResidualReturnsAfterOneMaterializedBatch) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_EQ(fake_state->get_block_count, 1); + EXPECT_EQ(fake_state->close_count, 0); + EXPECT_TRUE(reader.last_materialized_block_stats().has_materialized_input); + EXPECT_EQ(reader.last_materialized_block_stats().rows, 2); + EXPECT_GT(reader.last_materialized_block_stats().bytes, 0); + EXPECT_GT(reader.last_materialized_block_stats().allocated_bytes, 0); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, LateConjunctFiltersAlreadyOpenSplit) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + + bool predicate_executed = false; + auto late_predicate = std::make_shared(&predicate_executed); + late_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + ASSERT_TRUE( + reader.append_conjuncts({VExprContext::create_shared(std::move(late_predicate))}).ok()); + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->get_block_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + bool predicate_executed = false; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + ASSERT_NE(profile.get_counter("ResidualFilterTime"), nullptr); + EXPECT_GT(profile.get_counter("ResidualFilterTime")->value(), 0); ASSERT_TRUE(reader.close().ok()); } @@ -1471,9 +1733,11 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { EXPECT_EQ(fake_state->open_count, 1); ASSERT_NE(fake_state->last_request, nullptr); // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the scanner's row-level path instead of running inside the + // The later predicate must stay on the post-materialization path instead of running inside the // file reader before the unsafe conjunct. EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -1756,8 +2020,13 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { // presence still prevents the fake aggregate count (3) from replacing the two physical rows. ASSERT_NE(fake_state->last_request, nullptr); EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); - EXPECT_EQ(block.rows(), 2); + // The two physical rows are then filtered at the table boundary, where slotless predicates are + // evaluated exactly even though they cannot be localized to a file column. + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -4306,6 +4575,59 @@ TEST(TableReaderTest, VExprPredicateSurvivesReopenSplit) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_split_local_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto local_file = (test_dir / "local.parquet").string(); + const auto missing_file = (test_dir / "missing.parquet").string(); + write_single_int_parquet_file(local_file, "id", 3); + write_single_int_parquet_file(missing_file, "other", 9); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(local_file)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + expect_int32_column_values(*block.get_by_position(0).column, {3}); + ASSERT_TRUE(reader.close().ok()); + + // The same predicate cannot be file-local when this split omits `id`. It must be rebuilt as a + // table-level predicate over the materialized NULL instead of inheriting the previous split's + // file-local ownership or escaping without exact evaluation. + ASSERT_TRUE(reader.prepare_split(build_split_options(missing_file)).ok()); + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { const auto int_type = std::make_shared(); const std::vector projected_columns = { diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp index 60b6f37b8ceeba..64795dace19a14 100644 --- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp +++ b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp @@ -89,6 +89,18 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), expected_bpr); } +TEST_F(AdaptiveBlockSizePredictorTest, ExplicitMaterializedSampleUsesPreFilterShape) { + AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0); + + // Callers that filter a block before returning it can still report the rows and bytes that + // were actually materialized upstream. + pred.update(32, 32 * 4096); + + EXPECT_TRUE(pred.has_history_for_test()); + EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), 4096.0); + EXPECT_EQ(pred.predict_next_rows(), 2048); +} + // ── Test 2: EWMA convergence ────────────────────────────────────────────────── // When every update delivers the same sample, the EWMA stays exactly at that // value (0.9*v + 0.1*v == v for any v). From f50b4eb45cf9416ed24ed493f77a5ca9487656e2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 18:44:19 +0800 Subject: [PATCH 21/24] [fix](test) Use branch-native runtime filter wrapper in scanner test --- be/test/exec/scan/scanner_late_arrival_rf_test.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index b70f1cf9ea0a18..c926f3dc58d595 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -29,6 +29,9 @@ #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" #include "exec/scan/scanner.h" +// branch-4.1 creates VRuntimeFilterWrapper before the RuntimeFilterExpr rename, so assert the +// branch-native wrapper type while preserving the late-arrival batching invariant. +#include "exprs/vruntimefilter_wrapper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "testutil/column_helper.h" @@ -158,7 +161,7 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_EQ(scanner->_applied_rf_num, 2); ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { - EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); } ASSERT_EQ(scanner->_conjuncts.size(), 3); EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); @@ -168,7 +171,7 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(second_scanner->_applied_rf_num, 2); ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); - EXPECT_NE(dynamic_cast( + EXPECT_NE(dynamic_cast( second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), nullptr); From e191594296ac9e8d4bf760c71605f4b715dee678 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 21:47:46 +0800 Subject: [PATCH 22/24] [fix](file) Resolve remaining FileScannerV2 audit issues (#65931) ### What problem does this PR solve? This follow-up audits the 10 unresolved review threads on https://github.com/apache/doris/pull/65674 and every child of DORIS-27038. - Three unresolved threads were already fixed on master. - This PR fixes the remaining seven review findings. ### What is changed? - Harden Parquet delta geometry, page allocation validation, schema ambiguity handling, dictionary reuse, decompression scratch lifetime, and one-child MAP_KEY_VALUE SET parsing. - Validate delete expression result ownership before erasing temporary columns. - Scope Iceberg row-lineage virtual columns to Iceberg readers. - Propagate Remote Doris Flight timeout and cancellation. - Reject NULL JDBC special-type casts for non-nullable targets. - Aggregate hybrid Paimon/Hudi condition-cache hit counters. - Remove redundant JSON-line copies and Hive key allocations. The scanner V1 path under `be/src/format` is intentionally untouched. ### Verification - BE ASAN unit tests: 172 tests from 12 related suites passed. - clang-format 16 dry-run passed for every changed C++ file. - `git diff --check` passed. - Confirmed no diff under `be/src/format`. --------- Signed-off-by: Gabriel --- be/src/format_v2/column_mapper.cpp | 9 +- be/src/format_v2/column_mapper.h | 1 + be/src/format_v2/jni/jdbc_reader.cpp | 15 + be/src/format_v2/jni/jdbc_reader.h | 2 + be/src/format_v2/json/json_reader.cpp | 54 ++- be/src/format_v2/json/json_reader.h | 44 +- .../format_v2/parquet/native_schema_desc.cpp | 16 +- .../format_v2/parquet/native_schema_node.cpp | 35 +- be/src/format_v2/parquet/parquet_scan.cpp | 23 +- be/src/format_v2/parquet/parquet_scan.h | 2 + .../reader/native/column_chunk_reader.cpp | 162 +++++++- .../reader/native/column_chunk_reader.h | 32 +- .../parquet/reader/native/column_reader.cpp | 16 +- .../parquet/reader/native/column_reader.h | 6 + .../reader/native/delta_bit_pack_decoder.h | 6 + .../parquet/reader/native_column_reader.cpp | 19 +- .../parquet/reader/native_column_reader.h | 1 + be/src/format_v2/table/hudi_reader.cpp | 7 + be/src/format_v2/table/hudi_reader.h | 5 + ...eberg_position_delete_sys_table_reader.cpp | 1 + be/src/format_v2/table/iceberg_reader.h | 1 + be/src/format_v2/table/paimon_reader.cpp | 7 + be/src/format_v2/table/paimon_reader.h | 5 + .../format_v2/table/remote_doris_reader.cpp | 185 ++++++++- be/src/format_v2/table/remote_doris_reader.h | 2 + be/src/format_v2/table_reader.h | 3 +- be/test/format_v2/column_mapper_test.cpp | 28 +- be/test/format_v2/jni/jdbc_reader_test.cpp | 49 +++ be/test/format_v2/json/json_reader_test.cpp | 24 +- .../format_v2/parquet/native_decoder_test.cpp | 381 +++++++++++++++++- .../format_v2/parquet/parquet_scan_test.cpp | 50 +++ .../format_v2/parquet/parquet_schema_test.cpp | 112 +++++ be/test/format_v2/table/hudi_reader_test.cpp | 7 + .../format_v2/table/paimon_reader_test.cpp | 7 + .../table/remote_doris_reader_test.cpp | 269 +++++++++++++ 35 files changed, 1511 insertions(+), 75 deletions(-) create mode 100644 be/test/format_v2/jni/jdbc_reader_test.cpp diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 8ac3060bac48ab..ed0dcda9739d6b 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -395,7 +395,7 @@ std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection - << "}"; + << ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns << "}"; return out.str(); } @@ -2031,7 +2031,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab mapping->global_index = global_index; mapping->table_column_name = table_column.name; mapping->table_type = table_column.type; - const auto row_lineage_type = row_lineage_virtual_column_type(table_column, _options.mode); + // Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive, + // Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis. + const auto row_lineage_type = + _options.enable_row_lineage_virtual_columns + ? row_lineage_virtual_column_type(table_column, _options.mode) + : TableVirtualColumnType::INVALID; if (const auto* partition_value = find_partition_value(table_column, _partition_values); table_column.is_partition_key && partition_value != nullptr) { // Partition values are split constants and must take precedence over defaults. diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 1ee306a87741de..0860b687e4abc6 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -167,6 +167,7 @@ struct ColumnMapping { struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; bool allow_idless_complex_wrapper_projection = false; + bool enable_row_lineage_virtual_columns = false; std::string debug_string() const; }; diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index 7d28134db4d7e3..41d3a5282b6255 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -34,6 +34,20 @@ namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows) { + const auto* nullable = check_and_get_column(&result); + if (UNLIKELY(nullable == nullptr)) { + return Status::InternalError("JDBC special-type CAST did not return a nullable column"); + } + if (UNLIKELY(nullable->has_null(0, rows))) { + // CAST NULL represents invalid source data; stripping the null map would turn it into a + // valid-looking default for a NOT NULL destination. + return Status::DataQualityError( + "JDBC special-type CAST produced NULL for a non-nullable column"); + } + return Status::OK(); +} + std::string JdbcJniReader::connector_class() const { return "org/apache/doris/jdbc/JdbcJniScanner"; } @@ -184,6 +198,7 @@ Status JdbcJniReader::_cast_string_to_special_type(const format::JniTableReader: if (target_type->is_nullable()) { output_block->replace_by_position(column.output_index, result_column); } else { + RETURN_IF_ERROR(validate_non_nullable_special_type_result(*result_column, rows)); const auto* nullable_column = assert_cast(result_column.get()); output_block->replace_by_position(column.output_index, nullable_column->get_nested_column_ptr()); diff --git a/be/src/format_v2/jni/jdbc_reader.h b/be/src/format_v2/jni/jdbc_reader.h index 91a5878cb4622f..e10f8ad97902b8 100644 --- a/be/src/format_v2/jni/jdbc_reader.h +++ b/be/src/format_v2/jni/jdbc_reader.h @@ -29,6 +29,8 @@ namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows); + class JdbcJniReader final : public format::JniTableReader { public: ~JdbcJniReader() override = default; diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 1f42a2cd2d5e94..caf4205fa61c5b 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -270,10 +270,16 @@ Status JsonReader::open(std::shared_ptr request) { DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); _slot_name_to_index.clear(); + _hive_slot_name_to_index.clear(); _slot_name_to_index.reserve(_requested_columns.size()); + _hive_slot_name_to_index.reserve(_requested_columns.size()); for (size_t idx = 0; idx < _requested_columns.size(); ++idx) { auto name = _requested_columns[idx].slot_desc->col_name(); - _slot_name_to_index.emplace(_is_hive_table ? lower_key(name) : name, idx); + if (_is_hive_table) { + _hive_slot_name_to_index.emplace(std::move(name), idx); + } else { + _slot_name_to_index.emplace(std::move(name), idx); + } } _previous_positions.clear(); _reader_range = _json_range(); @@ -532,7 +538,9 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { if (*eof) { return Status::OK(); } - _document_buffer.assign(reinterpret_cast(line), *size); + // The line reader owns this span until the next read, and parsing copies it immediately + // into the padded buffer. Borrowing it avoids a redundant line-sized string copy. + _document_view = std::string_view(reinterpret_cast(line), *size); return Status::OK(); } // Non-line mode treats the split as one JSON document. This supports a single object or an @@ -558,6 +566,7 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { Slice result(_document_buffer.data(), _document_buffer.size()); RETURN_IF_ERROR(_physical_file_reader->read_at(_current_offset, result, size, _io_ctx.get())); _document_buffer.resize(*size); + _document_view = _document_buffer; if (*size == 0) { *eof = true; } @@ -572,6 +581,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { DorisUniqueBufferPtr file_buf; RETURN_IF_ERROR(stream_load_pipe->read_one_message(&file_buf, read_size)); _document_buffer.assign(reinterpret_cast(file_buf.get()), *read_size); + _document_view = _document_buffer; if (!stream_load_pipe->is_chunked_transfer()) { return Status::OK(); } @@ -586,6 +596,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { _document_buffer.append(reinterpret_cast(next_buf.get()), next_size); *read_size += next_size; } + _document_view = _document_buffer; return Status::OK(); } @@ -594,10 +605,10 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { if (*eof || *size == 0) { return Status::OK(); } - if (*size >= 3 && static_cast(_document_buffer[0]) == 0xEF && - static_cast(_document_buffer[1]) == 0xBB && - static_cast(_document_buffer[2]) == 0xBF) { - _document_buffer.erase(0, 3); + if (*size >= 3 && static_cast(_document_view[0]) == 0xEF && + static_cast(_document_view[1]) == 0xBB && + static_cast(_document_view[2]) == 0xBF) { + _document_view.remove_prefix(3); *size -= 3; } if (*size + simdjson::SIMDJSON_PADDING > _padded_size) { @@ -606,7 +617,7 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { } // Ondemand values reference the input buffer. Keep the padded bytes in a member buffer until the // current document is fully materialized. - std::memcpy(_padding_buffer.data(), _document_buffer.data(), *size); + std::memcpy(_padding_buffer.data(), _document_view.data(), *size); _original_doc_size = *size; const auto error = _json_parser->iterate(std::string_view(_padding_buffer.data(), *size), _padded_size) @@ -1120,32 +1131,39 @@ void JsonReader::_pop_back_last_inserted_value(Block* block, size_t column_index } size_t JsonReader::_column_index(std::string_view key, size_t key_index) { - std::string hive_key; - std::string_view lookup_key = key; - if (_is_hive_table) { - hive_key = lower_key(key); - lookup_key = hive_key; - } if (key_index < _previous_positions.size()) { // Most JSON lines share field order. Reuse the previous line's key-position mapping before // falling back to the hash table lookup. const auto previous = _previous_positions[key_index]; if (previous < _requested_columns.size()) { const auto previous_name = _requested_columns[previous].slot_desc->col_name(); - if ((_is_hive_table ? lower_key(previous_name) : previous_name) == lookup_key) { + if ((_is_hive_table && CaseInsensitiveStringEqual {}(previous_name, key)) || + (!_is_hive_table && previous_name == key)) { return previous; } } } - const auto it = _slot_name_to_index.find(std::string(lookup_key)); - if (it == _slot_name_to_index.end()) { + // Transparent lookup keeps the common per-key path allocation-free; Hive case folding is + // performed by the map's hash/equality without constructing a lower-cased string. + const auto index = _is_hive_table ? ([&]() -> std::optional { + const auto it = _hive_slot_name_to_index.find(key); + return it == _hive_slot_name_to_index.end() ? std::nullopt + : std::optional(it->second); + })() + : ([&]() -> std::optional { + const auto it = _slot_name_to_index.find(key); + return it == _slot_name_to_index.end() + ? std::nullopt + : std::optional(it->second); + })(); + if (!index.has_value()) { return static_cast(-1); } if (key_index >= _previous_positions.size()) { _previous_positions.resize(key_index + 1, static_cast(-1)); } - _previous_positions[key_index] = it->second; - return it->second; + _previous_positions[key_index] = *index; + return *index; } bool JsonReader::_is_root_path_for_column(const RequestedColumn& column) const { diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index de3c084e681f23..c7346cb1d66357 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -42,6 +42,40 @@ class IColumn; namespace doris::format::json { +struct TransparentStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + return std::hash {}(value); + } +}; + +struct CaseInsensitiveStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + size_t hash = 1469598103934665603ULL; + for (const unsigned char c : value) { + const unsigned char folded = c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + hash = (hash ^ folded) * 1099511628211ULL; + } + return hash; + } +}; + +struct CaseInsensitiveStringEqual { + using is_transparent = void; + bool operator()(std::string_view lhs, std::string_view rhs) const { + if (lhs.size() != rhs.size()) return false; + for (size_t i = 0; i < lhs.size(); ++i) { + const unsigned char left = + lhs[i] >= 'A' && lhs[i] <= 'Z' ? lhs[i] + ('a' - 'A') : lhs[i]; + const unsigned char right = + rhs[i] >= 'A' && rhs[i] <= 'Z' ? rhs[i] + ('a' - 'A') : rhs[i]; + if (left != right) return false; + } + return true; + } +}; + // FileScannerV2 JSON reader. // // JSON files do not carry an embedded physical schema. The v2 table layer still needs a @@ -74,6 +108,10 @@ class JsonReader final : public FileReader { Status get_block(Block* file_block, size_t* rows, bool* eof) override; Status close() override; +#ifdef BE_TEST + size_t TEST_document_buffer_size() const { return _document_buffer.size(); } +#endif + private: void _init_profile() override; // A requested column keeps both identities: @@ -135,7 +173,10 @@ class JsonReader final : public FileReader { TFileCompressType::type _range_compress_type = TFileCompressType::UNKNOWN; std::optional _stream_load_id; std::vector _requested_columns; - std::unordered_map _slot_name_to_index; + std::unordered_map> + _slot_name_to_index; + std::unordered_map + _hive_slot_name_to_index; std::vector _previous_positions; RuntimeProfile::Counter* _total_time = nullptr; @@ -178,6 +219,7 @@ class JsonReader final : public FileReader { simdjson::ondemand::array _array; simdjson::ondemand::array_iterator _array_iter; std::string _document_buffer; + std::string_view _document_view; std::string _padding_buffer; size_t _original_doc_size = 0; size_t _padded_size = 1024 * 1024 * 8 + simdjson::SIMDJSON_PADDING; diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 713707b84b8e79..9eba6d8feaa55a 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -207,10 +207,6 @@ static bool is_struct_list_node(const tparquet::SchemaElement& schema, return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; } -static bool has_logical_annotation(const tparquet::SchemaElement& schema) { - return schema.__isset.logicalType || schema.__isset.converted_type; -} - std::string NativeFieldSchema::debug_string() const { std::stringstream ss; ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level @@ -625,9 +621,17 @@ Status NativeFieldDescriptor::parse_list_field( if (num_children > 0) { const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); const auto& only_child = t_schemas[curr_pos + 2]; - if (num_children == 1 && !structural_wrapper && has_logical_annotation(second_level)) { + const bool single_key_set_wrapper = + second_level.__isset.converted_type && + second_level.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE && + !is_repeated_node(only_child); + const bool nested_collection_annotation = + is_list_node(second_level) || + (is_map_node(second_level) && !single_key_set_wrapper); + if (num_children == 1 && !structural_wrapper && nested_collection_annotation) { // The repeated node is already the outer LIST element. Preserve its own LIST/MAP - // annotation, but do not interpret its REPEATED marker as another outer array. + // annotation. A MAP_KEY_VALUE node with a repeated child is the legacy uncontained + // MAP shape; only its direct, non-repeated child denotes the enclosing SET key. set_child_node_level(list_field, list_field->definition_level); if (is_list_node(second_level)) { RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp index 7539fb45efb46d..052df93e4b4143 100644 --- a/be/src/format_v2/parquet/native_schema_node.cpp +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -68,24 +68,39 @@ Status build_native_schema_node(const DataTypePtr& projected_type, return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); } const auto* struct_type = assert_cast(type.get()); - std::map file_children; - for (const auto& child : file_schema.children) { - file_children.emplace(to_lower(child->name), child.get()); - } auto node = std::make_shared(); for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { const auto& table_name = struct_type->get_element_name(i); - // Native metadata keeps writer casing. Match normalized names while preserving the - // original file name used to address the physical child reader. - const auto child_it = file_children.find(to_lower(table_name)); - if (child_it == file_children.end()) { + const ParquetColumnSchema* file_child = nullptr; + for (const auto& child : file_schema.children) { + if (child->name == table_name) { + file_child = child.get(); + break; + } + } + if (file_child == nullptr) { + for (const auto& child : file_schema.children) { + if (to_lower(child->name) != to_lower(table_name)) { + continue; + } + if (UNLIKELY(file_child != nullptr)) { + // Exact writer identity is authoritative; normalized fallback is only safe + // when the requested field has one physical candidate. + return Status::Corruption( + "Parquet STRUCT {} has ambiguous case-insensitive child name {}", + file_schema.name, table_name); + } + file_child = child.get(); + } + } + if (file_child == nullptr) { node->add_missing_child(table_name); continue; } std::shared_ptr child_node; - RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *child_it->second, + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *file_child, &child_node)); - node->add_child(table_name, child_it->second->name, std::move(child_node)); + node->add_child(table_name, file_child->name, std::move(child_node)); } *result = std::move(node); return Status::OK(); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 47e7c4c3a68af2..b92a3c27054ad1 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -652,11 +652,12 @@ Status execute_compact_delete_conjuncts(const VExprContextSPtrs& delete_conjunct *can_filter_all = false; for (const auto& delete_conjunct : delete_conjuncts) { DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -702,11 +703,12 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t break; } DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -727,6 +729,19 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t } // namespace +Status detail::validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns) { + // Delete predicates may erase only a temporary expression result. A bare SlotRef returns an + // input column id, which must remain in the block for later predicates and materialization. + if (UNLIKELY(result_column_id < 0 || static_cast(result_column_id) < original_columns || + static_cast(result_column_id) >= current_columns)) { + return Status::InternalError( + "Delete conjunct result column {} is not ephemeral (original={}, current={})", + result_column_id, original_columns, current_columns); + } + return Status::OK(); +} + uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, uint16_t selected_rows) { DORIS_CHECK(selection != nullptr); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index dc926c3a7666e7..ac883cbbcbe099 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -77,6 +77,8 @@ std::vector adaptive_prefetch_prefix( const std::unordered_map& stats, double minimum_reach_probability); bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns); Status build_native_prefetch_ranges( const tparquet::FileMetaData& metadata, const std::vector>& file_schema, diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index c219a7aef03be2..26afaf15bf4c4c 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -86,6 +87,102 @@ Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, return Status::OK(); } +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required) { + if (type_length <= 0) { + return Status::OK(); + } + const bool is_v2 = header.__isset.data_page_header_v2; + if (!is_v2 && !header.__isset.data_page_header) { + return Status::OK(); + } + const auto encoding = + is_v2 ? header.data_page_header_v2.encoding : header.data_page_header.encoding; + if (encoding != tparquet::Encoding::PLAIN && + encoding != tparquet::Encoding::BYTE_STREAM_SPLIT) { + return Status::OK(); + } + int32_t num_physical_values = 0; + int64_t level_bytes = 0; + if (is_v2) { + const auto& page = header.data_page_header_v2; + if (UNLIKELY(page.num_values < 0 || page.num_nulls < 0 || + page.num_nulls > page.num_values || page.repetition_levels_byte_length < 0 || + page.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has invalid value or level counts"); + } + num_physical_values = page.num_values - page.num_nulls; + level_bytes = static_cast(page.repetition_levels_byte_length) + + page.definition_levels_byte_length; + } else { + if (max_rep_level != 0 || max_def_level != 0 || !schema_is_required) { + return Status::OK(); + } + num_physical_values = header.data_page_header.num_values; + } + if (level_bytes > std::numeric_limits::max() || num_physical_values < 0 || + static_cast(num_physical_values) > + (static_cast(std::numeric_limits::max()) - level_bytes) / + static_cast(type_length)) { + return Status::Corruption("Parquet fixed-width PLAIN page byte size overflows"); + } + const int64_t expected = level_bytes + static_cast(num_physical_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // V2 exposes null and level extents separately, so fixed-width payload size is known before + // decompression even for optional columns and must gate attacker-controlled allocation. + return Status::Corruption("Parquet fixed-width page has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + return Status::OK(); +} + +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length) { + DORIS_CHECK(header.__isset.dictionary_page_header); + const int32_t num_values = header.dictionary_page_header.num_values; + if (UNLIKELY(num_values < 0 || (num_values == 0 && header.uncompressed_page_size != 0))) { + // An empty dictionary owns no payload; validate before allocating from its untrusted size. + return Status::Corruption("Parquet dictionary has {} values and {} uncompressed bytes", + num_values, header.uncompressed_page_size); + } + if (type_length > 0) { + if (UNLIKELY(static_cast(num_values) > + static_cast(std::numeric_limits::max()) / + static_cast(type_length))) { + return Status::Corruption("Parquet fixed-width dictionary byte size overflows"); + } + const int64_t expected = static_cast(num_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // Fixed-width dictionaries have no level section, so reject forged extents before the + // decoder allocates storage based on the untrusted page header. + return Status::Corruption( + "Parquet fixed-width dictionary has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + } + return Status::OK(); +} + +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size) { + if (codec != tparquet::CompressionCodec::SNAPPY) { + return Status::OK(); + } + size_t actual_uncompressed_size = 0; + if (UNLIKELY(!snappy::GetUncompressedLength(compressed_data.data, compressed_data.size, + &actual_uncompressed_size))) { + return Status::Corruption("Invalid Snappy-compressed Parquet page"); + } + if (UNLIKELY(actual_uncompressed_size != expected_uncompressed_size)) { + // Snappy exposes its decoded extent without an output buffer. Check it before trusting the + // page header so malformed variable-width pages cannot force a header-sized allocation. + return Status::Corruption("Snappy Parquet page expands to {} bytes, expected {}", + actual_uncompressed_size, expected_uncompressed_size); + } + return Status::OK(); +} + ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { if (created_by.empty()) { return {}; @@ -905,6 +1002,18 @@ Status ColumnChunkReader::parse_page_header() { template Status ColumnChunkReader::next_page() { + // Level parsing advances _page_data past the allocation base, so retain explicit ownership + // state instead of inferring whether current decoders still reference decompressed storage. + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; + if (_decompress_release_pending) { + if (_decompress_buf_size > _decompress_release_threshold) { + _decompress_buf.reset(); + _decompress_buf_size = 0; + } + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } _state = INITIALIZED; RETURN_IF_ERROR(_page_reader->next_page()); return Status::OK(); @@ -939,8 +1048,17 @@ Status ColumnChunkReader::load_page_data() { RETURN_IF_ERROR(_page_reader->get_page_header(&header)); RETURN_IF_ERROR(validate_uncompressed_page_sizes( *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + // Zero levels alone are insufficient: test/protocol adapters can leave repetition unset, so + // only an explicitly REQUIRED schema proves that every logical value has fixed-width bytes. + const bool schema_is_required = _field_schema->parquet_schema.__isset.repetition_type && + _field_schema->parquet_schema.repetition_type == + tparquet::FieldRepetitionType::REQUIRED; + RETURN_IF_ERROR(validate_fixed_width_page_size(*header, _get_type_length(), _max_rep_level, + _max_def_level, schema_is_required)); int32_t uncompressed_size = header->uncompressed_page_size; bool page_loaded = false; + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; // First, try to reuse a cache handle previously discovered by PageReader // (header-only lookup) to avoid a second lookup here. @@ -992,8 +1110,12 @@ Status ColumnChunkReader::load_page_data() { header->__isset.data_page_header_v2 ? static_cast(header->uncompressed_page_size) - levels_size : static_cast(header->uncompressed_page_size); + RETURN_IF_ERROR(validate_compressed_page_size(_metadata.codec, payload_slice, + uncompressed_payload_size)); _reserve_decompress_buf(uncompressed_payload_size); _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = uncompressed_payload_size; SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); _chunk_statistics.decompress_cnt++; RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); @@ -1040,8 +1162,12 @@ Status ColumnChunkReader::load_page_data() { if (page_has_compression) { // Decompress payload for immediate decoding + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); _reserve_decompress_buf(uncompressed_size); _page_data = Slice(_decompress_buf.get(), uncompressed_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = static_cast(uncompressed_size); SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); _chunk_statistics.decompress_cnt++; RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); @@ -1186,7 +1312,15 @@ Status ColumnChunkReader::_decode_dict_page() { int32_t uncompressed_size = header->uncompressed_page_size; RETURN_IF_ERROR(validate_uncompressed_page_sizes( *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); - auto dict_data = make_unique_buffer(uncompressed_size); + RETURN_IF_ERROR(validate_dictionary_page_size(*header, _get_type_length())); + DorisUniqueBufferPtr dict_data; + bool dict_buffer_allocated = false; + const auto allocate_dict_buffer = [&]() { + if (!dict_buffer_allocated) { + dict_data = make_unique_buffer(uncompressed_size); + dict_buffer_allocated = true; + } + }; bool dict_loaded = false; // Try to load dictionary page from cache @@ -1196,6 +1330,10 @@ Status ColumnChunkReader::_decode_dict_page() { const PageCacheHandle& handle = _page_reader->page_cache_handle(); Slice cached = handle.data(); size_t header_size = _page_reader->header_bytes().size(); + if (UNLIKELY(header_size > cached.size)) { + return Status::Corruption( + "Cached Parquet dictionary is shorter than its page header"); + } // Dictionary page layout in cache: header | payload (compressed or uncompressed) Slice payload_slice(cached.data + header_size, cached.size - header_size); @@ -1208,11 +1346,15 @@ Status ColumnChunkReader::_decode_dict_page() { "Cached Parquet dictionary payload has size {}, expected {}", payload_slice.size, uncompressed_size); } + allocate_dict_buffer(); memcpy(dict_data.get(), payload_slice.data, payload_slice.size); dict_loaded = true; } else { CHECK(_block_compress_codec); // Decompress cached compressed dictionary data + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, payload_slice, static_cast(uncompressed_size))); + allocate_dict_buffer(); Slice dict_slice(dict_data.get(), uncompressed_size); { SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); @@ -1239,15 +1381,13 @@ Status ColumnChunkReader::_decode_dict_page() { // Load and decompress dictionary page from file if (_block_compress_codec != nullptr) { auto dict_num = header->dictionary_page_header.num_values; - if (dict_num == 0 && uncompressed_size != 0) { - return Status::IOError( - "Dictionary page's num_values is {} but uncompressed_size is {}", dict_num, - uncompressed_size); - } Slice compressed_data; - Slice dict_slice(dict_data.get(), uncompressed_size); if (dict_num != 0) { RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); // Dictionary probes stop before data pages, so count their decompression here or // metadata pruning profiles will report no codec work for the scan. { @@ -1262,6 +1402,8 @@ Status ColumnChunkReader::_decode_dict_page() { dict_slice.size, uncompressed_size); } } + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); // Decide whether to cache decompressed or compressed dictionary based on threshold // If uncompressed_page_size == 0, should_cache_decompressed will return true @@ -1295,6 +1437,11 @@ Status ColumnChunkReader::_decode_dict_page() { } else { Slice dict_slice; RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet dictionary payload has size {}, expected {}", + dict_slice.size, uncompressed_size); + } + allocate_dict_buffer(); // The data is stored by BufferedStreamReader, we should copy it out memcpy(dict_data.get(), dict_slice.data, dict_slice.size); @@ -1309,6 +1456,7 @@ Status ColumnChunkReader::_decode_dict_page() { } } } + allocate_dict_buffer(); // Cache page decoder std::unique_ptr page_decoder; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 5acf544bcc95da..102366f24949ee 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -19,8 +19,10 @@ #include +#include #include #include +#include #include #include #include @@ -71,6 +73,13 @@ bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cac Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, tparquet::CompressionCodec::type codec, bool data_page_v2_always_compressed); +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required = true); +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length = -1); +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size); struct ColumnChunkReaderStatistics { int64_t decompress_time = 0; @@ -215,10 +224,24 @@ class ColumnChunkReader { // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. _rep_level_decoder.release_scratch(max_retained_bytes); _def_level_decoder.release_scratch(max_retained_bytes); + if (_decompress_buf_size > max_retained_bytes) { + if (_page_uses_decompress_buf) { + // Keep the request until the page boundary because decoders still point into this + // allocation; dropping it now would trade retained memory for a use-after-free. + _decompress_release_pending = true; + _decompress_release_threshold = + std::min(_decompress_release_threshold, max_retained_bytes); + } else { + _decompress_buf.reset(); + _decompress_buf_size = 0; + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } + } } size_t retained_decoder_scratch_bytes() const { - size_t bytes = _rep_level_decoder.retained_scratch_bytes() + + size_t bytes = _decompress_buf_size + _rep_level_decoder.retained_scratch_bytes() + _def_level_decoder.retained_scratch_bytes(); for (const auto& [encoding, decoder] : _decoders) { bytes += decoder->retained_scratch_bytes(); @@ -229,7 +252,8 @@ class ColumnChunkReader { size_t active_decoder_scratch_bytes() const { // Only the current encoding is active. Old decoder instances retain reusable capacity but // must not make the high-water policy treat their last batch as current working memory. - return (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + return _active_decompress_bytes + + (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + _rep_level_decoder.active_scratch_bytes() + _def_level_decoder.active_scratch_bytes(); } @@ -367,6 +391,10 @@ class ColumnChunkReader { Slice _page_data; DorisUniqueBufferPtr _decompress_buf; size_t _decompress_buf_size = 0; + bool _page_uses_decompress_buf = false; + size_t _active_decompress_bytes = 0; + bool _decompress_release_pending = false; + size_t _decompress_release_threshold = std::numeric_limits::max(); Slice _v2_rep_levels; Slice _v2_def_levels; bool _dict_checked = false; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 9417da15683c87..592e048846e137 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -1271,6 +1271,10 @@ ScalarColumnReader::materialize_dictionary_values( // interpretation as ordinary data-page decoding. const DataTypePtr dictionary_type = remove_nullable(target_type); const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + // The probe is the first typed read for this reader. Publish its SerDe identity now so the + // first dictionary-ID batch does not mistake initialization for a type change and drop cache. + _serde_type = dictionary_type.get(); + _serde = dictionary_serde; if (_materialization_state.dictionary_generation != dictionary_decoder->dictionary_generation()) { _materialization_state.typed_dictionary = dictionary_type->create_column(); @@ -1285,6 +1289,9 @@ ScalarColumnReader::materialize_dictionary_values( DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), dictionary_decoder->dictionary_size()); _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); +#ifdef BE_TEST + ++_dictionary_materialization_count; +#endif } auto result = _materialization_state.typed_dictionary->clone_empty(); @@ -1350,11 +1357,16 @@ Status ScalarColumnReader::read_column_data( } const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; - if (_serde_type != serde_type.get() || _dictionary_index_only != is_dict_filter) { + const bool serde_type_changed = _serde_type != serde_type.get(); + if (serde_type_changed || _dictionary_index_only != is_dict_filter) { _serde_type = serde_type.get(); _serde = serde_type->get_serde(); _dictionary_index_only = is_dict_filter; - _materialization_state.reset_dictionary(); + // Switching between typed values and dictionary IDs does not invalidate the dictionary + // materialized by the pruning probe. Preserve it unless the logical SerDe type changed. + if (serde_type_changed) { + _materialization_state.reset_dictionary(); + } } _decode_context.dictionary_index_only = is_dict_filter; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index b9a25a8132ed91..6ea2787f9f0211 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -300,6 +300,9 @@ class ScalarColumnReader : public ColumnReader { #ifdef BE_TEST void reserve_batch_scratch_for_test(size_t elements); size_t retained_batch_scratch_bytes_for_test() const; + size_t dictionary_materialization_count_for_test() const { + return _dictionary_materialization_count; + } #endif void reset_filter_map_index() override { @@ -311,6 +314,9 @@ class ScalarColumnReader : public ColumnReader { const tparquet::OffsetIndex* _offset_index = nullptr; std::unique_ptr _stream_reader; std::unique_ptr> _chunk_reader; +#ifdef BE_TEST + size_t _dictionary_materialization_count = 0; +#endif // rep def levels buffer. std::vector _rep_levels; std::vector _def_levels; diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index 531cb7704075b5..db9efecaced470 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -672,6 +672,12 @@ Status DeltaBitPackDecoder::_init_header() { if (_mini_blocks_per_block == 0) { return Status::InvalidArgument("Cannot have zero miniblock per block"); } + // Parquet requires integral block geometry; truncating here would silently decode fewer + // values than the header declares and desynchronize the following block. + if (UNLIKELY(_values_per_block % _mini_blocks_per_block != 0)) { + return Status::Corruption("Parquet delta block size {} is not divisible by {} miniblocks", + _values_per_block, _mini_blocks_per_block); + } _values_per_mini_block = _values_per_block / _mini_blocks_per_block; if (_values_per_mini_block == 0) { return Status::InvalidArgument("Cannot have zero value per miniblock"); diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 8aa79ecb7e7e23..233b37c096e300 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -323,13 +323,7 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ if (_nested && _profile.nested_batches != nullptr) { COUNTER_UPDATE(_profile.nested_batches, 1); } - // Retained-capacity inspection walks the native reader tree. Check it periodically instead of - // on every small batch; row-group destruction is still the hard lifetime bound for scratch. - constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; - if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { - _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); - _batches_since_scratch_check = 0; - } + release_batch_scratch_if_needed(); if (*rows_read != rows) { return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", *rows_read, rows, _name); @@ -397,9 +391,20 @@ Status NativeColumnReader::read_with_fixed_width_filter(int64_t rows, const uint *rows_read, rows, _name); } *used_filter = true; + release_batch_scratch_if_needed(); return Status::OK(); } +void NativeColumnReader::release_batch_scratch_if_needed() { + // PLAIN predicate batches bypass materialization but share the same persistent decoder tree, + // so both read paths must advance the retained-capacity aging clock. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } +} + Status NativeColumnReader::validate_selected_span(int64_t rows) { DORIS_CHECK(rows >= 0); while (_selected_range_idx < _selected_ranges.size()) { diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 0ffb8d054008b7..1d92735bfa19d3 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -116,6 +116,7 @@ class NativeColumnReader final : public ParquetColumnReader { const VExprSPtrs& conjuncts, int column_id, IColumn* projected_column, IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter); + void release_batch_scratch_if_needed(); int64_t sync_native_profile(); void record_page_fragments(int64_t page_fragments); Status validate_selected_span(int64_t rows); diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index ef9c277254311f..271f14a7b483eb 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -146,6 +146,13 @@ const format::MaterializedBlockStats& HudiHybridReader::last_materialized_block_ : format::TableReader::last_materialized_block_stats(); } +int64_t HudiHybridReader::condition_cache_hit_count() const { + // Keep the wrapper count cumulative across native/JNI dispatch so scanner-level delta + // accounting neither loses a child hit nor observes a counter reset on a split switch. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index 42776422c4ad19..c13ac1215d72ee 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -67,6 +67,7 @@ class HudiHybridReader final : public format::TableReader { void set_batch_size(size_t batch_size) override; Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST void TEST_install_batch_size_children() { @@ -76,6 +77,10 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } void TEST_set_child_reader_factories( std::function()> native_factory, std::function()> jni_factory) { diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index 16c7f829089e76..6ab67caa77da03 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -146,6 +146,7 @@ class PositionDeleteFileTableReader final : public format::TableReader { } void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; // Parquet may preserve a selected complex wrapper without its own ID; position-delete row // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. options->allow_idless_complex_wrapper_projection = diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index cf864188074dcd..d28be3d7f98f0b 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -69,6 +69,7 @@ class IcebergTableReader : public format::TableReader { protected: void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; options->allow_idless_complex_wrapper_projection = supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET; } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 3e409fee9da5ce..d815ef81c9b424 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -173,6 +173,13 @@ const format::MaterializedBlockStats& PaimonHybridReader::last_materialized_bloc : format::TableReader::last_materialized_block_stats(); } +int64_t PaimonHybridReader::condition_cache_hit_count() const { + // Both children survive split switches, so the wrapper must publish their cumulative totals; + // returning only the active child would make FileScannerV2's monotonic delta go backwards. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 6ee5e7d72f3405..b4f076fc6e4878 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -73,6 +73,7 @@ class PaimonHybridReader final : public format::TableReader { void set_batch_size(size_t batch_size) override; Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); } @@ -87,6 +88,10 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } void TEST_set_child_reader_factories( std::function()> native_factory, std::function()> jni_factory) { diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index db278d9b0cedda..bd2c08b5f2310f 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -20,8 +20,13 @@ #include #include +#include +#include +#include #include +#include #include +#include #include #include @@ -38,7 +43,9 @@ #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" #include "runtime/file_scan_profile.h" +#include "runtime/query_context.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" #include "util/timezone_utils.h" namespace doris::format::remote_doris { @@ -64,7 +71,12 @@ Status validate_remote_doris_range(const TFileRangeDesc& range) { class FlightRemoteDorisStream final : public RemoteDorisStream { public: - explicit FlightRemoteDorisStream(const TFileRangeDesc& range) : _range(range) {} + FlightRemoteDorisStream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds) + : _range(range), + _io_ctx(std::move(io_ctx)), + _runtime_state(runtime_state), + _timeout_seconds(std::max(1, timeout_seconds)) {} Status open() { RETURN_IF_ERROR(validate_remote_doris_range(_range)); @@ -75,14 +87,103 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { arrow::flight::Ticket ticket; RETURN_DORIS_STATUS_IF_ERROR( arrow::flight::Ticket::Deserialize(params.ticket).Value(&ticket)); + struct PendingOpen { + std::mutex mutex; + std::condition_variable cv; + bool done = false; + bool abandoned = false; + arrow::Status status = arrow::Status::OK(); + std::unique_ptr client; + std::unique_ptr stream; + }; + auto pending = std::make_shared(); + std::unique_ptr flight_client; RETURN_DORIS_STATUS_IF_ERROR( - arrow::flight::FlightClient::Connect(location).Value(&_flight_client)); - RETURN_DORIS_STATUS_IF_ERROR(_flight_client->DoGet(ticket).Value(&_stream)); + arrow::flight::FlightClient::Connect(location).Value(&flight_client)); + arrow::flight::FlightCallOptions options; + // A Flight deadline covers streaming reads as well as DoGet setup, so a stalled Next() + // cannot outlive the query execution timeout indefinitely. + options.timeout = std::chrono::seconds(_timeout_seconds); + // Start before DoGet because endpoint setup is itself a blocking RPC covered by the same + // query/scanner cancellation contract as streaming Next(). + _cancellation_watcher = std::jthread( + [this](std::stop_token stop_token) { _watch_cancellation(stop_token); }); + + std::shared_ptr resource_ctx; + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr) { + resource_ctx = _runtime_state->get_query_ctx()->resource_ctx(); + } + std::thread do_get_thread([pending, options, ticket, resource_ctx, + client = std::move(flight_client)]() mutable { + const auto do_get = [&] { + std::unique_ptr stream; + auto status = client->DoGet(options, ticket).Value(&stream); + { + std::lock_guard lock(pending->mutex); + if (!pending->abandoned) { + pending->status = std::move(status); + pending->client = std::move(client); + pending->stream = std::move(stream); + } else { + // A detached worker must release its query-owned Flight client + // before leaving the task attachment that accounts for it. + client.reset(); + } + pending->done = true; + } + pending->cv.notify_all(); + }; + if (resource_ctx != nullptr) { + SCOPED_ATTACH_TASK(resource_ctx); + do_get(); + } else { + SCOPED_INIT_THREAD_CONTEXT(); + do_get(); + } + }); + bool cancelled_during_open = false; + { + std::unique_lock lock(pending->mutex); + while (!pending->done && !_is_cancelled()) { + pending->cv.wait_for(lock, std::chrono::milliseconds(25)); + } + if (!pending->done) { + pending->abandoned = true; + cancelled_during_open = true; + } + } + if (cancelled_during_open) { + // Arrow 17 exposes no cancellable handle until DoGet returns. Detaching the bounded RPC + // keeps query/scanner shutdown prompt while the call is still capped by its deadline; + // the shared state owns all Arrow objects until that worker exits. + do_get_thread.detach(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } + do_get_thread.join(); + if (!pending->status.ok()) { + _stop_cancellation_watcher(); + RETURN_DORIS_STATUS_IF_ERROR(pending->status); + } + { + std::lock_guard lock(_flight_mutex); + _flight_client = std::move(pending->client); + _stream = std::move(pending->stream); + } + if (_is_cancelled()) { + _cancel_flight_call(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } return Status::OK(); } Status next(std::shared_ptr* batch) override { DORIS_CHECK(batch != nullptr); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _cancel_flight_call(); + return Status::Cancelled("Remote Doris Flight read was cancelled"); + } arrow::flight::FlightStreamChunk chunk; RETURN_DORIS_STATUS_IF_ERROR(_stream->Next().Value(&chunk)); *batch = chunk.data; @@ -90,6 +191,13 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } Status close() override { + _stop_cancellation_watcher(); + { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } _stream.reset(); if (_flight_client != nullptr) { RETURN_DORIS_STATUS_IF_ERROR(_flight_client->Close()); @@ -99,14 +207,65 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } private: + bool _is_cancelled() const { + return (_runtime_state != nullptr && _runtime_state->is_cancelled()) || + (_io_ctx != nullptr && _io_ctx->should_stop); + } + + void _cancel_flight_call() { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } + + void _watch_cancellation_loop(std::stop_token watcher_stop_token) { + while (!watcher_stop_token.stop_requested()) { + if (_is_cancelled()) { + _cancel_flight_call(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + } + + void _watch_cancellation(std::stop_token watcher_stop_token) { + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr && + _runtime_state->get_query_ctx()->resource_ctx() != nullptr) { + // The watcher is query-owned and may allocate in Arrow while signalling cancellation. + SCOPED_ATTACH_TASK(_runtime_state); + _watch_cancellation_loop(watcher_stop_token); + return; + } + // Metadata/tests can construct a RuntimeState without a QueryContext; initialize TLS there + // instead of violating AttachTask's non-null resource-context invariant. + SCOPED_INIT_THREAD_CONTEXT(); + _watch_cancellation_loop(watcher_stop_token); + } + + void _stop_cancellation_watcher() { + if (_cancellation_watcher.joinable()) { + _cancellation_watcher.request_stop(); + _cancellation_watcher.join(); + } + } + const TFileRangeDesc _range; + std::shared_ptr _io_ctx; + RuntimeState* _runtime_state; + int _timeout_seconds; + std::jthread _cancellation_watcher; + std::mutex _flight_mutex; std::unique_ptr _flight_client; std::unique_ptr _stream; }; -Status create_flight_stream(const TFileRangeDesc& range, std::unique_ptr* out) { +Status create_flight_stream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds, + std::unique_ptr* out) { DORIS_CHECK(out != nullptr); - auto stream = std::make_unique(range); + auto stream = std::make_unique(range, std::move(io_ctx), runtime_state, + timeout_seconds); RETURN_IF_ERROR(stream->open()); *out = std::move(stream); return Status::OK(); @@ -199,7 +358,10 @@ void RemoteDorisFileReader::_init_profile() { Status RemoteDorisFileReader::init(RuntimeState* state) { _init_profile(); SCOPED_TIMER(_total_time); - (void)state; + if (state != nullptr) { + _flight_timeout_seconds = std::max(1, state->execution_timeout()); + } + _runtime_state = state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); _eof = false; @@ -244,6 +406,14 @@ Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* e if (_stream == nullptr) { return Status::InternalError("Remote Doris v2 reader is not open"); } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + // Observe cancellation before entering a potentially blocking Flight read; the production + // stream also carries a query-bounded RPC deadline for cancellation arriving mid-read. + RETURN_IF_ERROR(close()); + *rows = 0; + *eof = true; + return Status::OK(); + } *rows = 0; *eof = false; @@ -288,7 +458,8 @@ Status RemoteDorisFileReader::_open_stream() { if (_stream_factory) { RETURN_IF_ERROR(_stream_factory(_range, &_stream)); } else { - RETURN_IF_ERROR(create_flight_stream(_range, &_stream)); + RETURN_IF_ERROR(create_flight_stream(_range, _io_ctx, _runtime_state, + _flight_timeout_seconds, &_stream)); } DORIS_CHECK(_stream != nullptr); return Status::OK(); diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index 79c37a29be4a06..3b4a03f4569396 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -90,6 +90,8 @@ class RemoteDorisFileReader final : public FileReader { RuntimeProfile::Counter* _io_time = nullptr; RuntimeProfile::Counter* _materialize_time = nullptr; RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeState* _runtime_state = nullptr; + int _flight_timeout_seconds = 300; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 47d442f7fc17f6..48e3d996ce22fd 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -226,6 +226,7 @@ class TableReader { size_t TEST_table_reader_owned_conjunct_count() const { return _table_reader_owned_conjunct_count; } + void TEST_set_condition_cache_hit_count(int64_t hits) { _condition_cache_hit_count = hits; } bool TEST_current_data_file_is_immutable() const { DORIS_CHECK(_current_task != nullptr); DORIS_CHECK(_current_task->data_file != nullptr); @@ -404,7 +405,7 @@ class TableReader { return Status::OK(); } - int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } + virtual int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } virtual std::string debug_string() const; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 6bfc06139e4850..d3428b6134a479 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2071,7 +2071,8 @@ TEST(ColumnMapperConstantTest, PartitionDefaultAndVirtualColumnsUseDedicatedBran {"dt", Field::create_field("2026-06-11")}, }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, partition_values, {}).ok()); ASSERT_EQ(mapper.mappings().size(), 5); @@ -2092,7 +2093,8 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { name_col("_last_updated_sequence_number", make_nullable(i64()), 2147483539), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 2); @@ -2125,6 +2127,25 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { std::vector({2147483540, 2147483539})); } +TEST(ColumnMapperConstantTest, GenericByNameKeepsRowLineageNamesPhysical) { + const std::vector table_schema = { + name_col("_row_id", make_nullable(i64())), + name_col("_last_updated_sequence_number", make_nullable(i64())), + }; + const std::vector file_schema = { + name_col("_row_id", make_nullable(i64()), 0), + name_col("_last_updated_sequence_number", make_nullable(i64()), 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(mapper.mappings()[0].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::COPY_DIRECTLY); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[1].filter_conversion, FilterConversionType::COPY_DIRECTLY); +} + TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMapping) { auto id_column = field_id_col("id", 1, make_nullable(i32())); auto row_id_column = field_id_col("renamed_row_id", 2147483540, make_nullable(i64())); @@ -2141,7 +2162,8 @@ TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMappi field_id_col("name", 2, make_nullable(str()), 1), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 3); diff --git a/be/test/format_v2/jni/jdbc_reader_test.cpp b/be/test/format_v2/jni/jdbc_reader_test.cpp new file mode 100644 index 00000000000000..5763a6deb00895 --- /dev/null +++ b/be/test/format_v2/jni/jdbc_reader_test.cpp @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jdbc_reader.h" + +#include + +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" + +namespace doris::format::jdbc { + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeRejectsCastNull) { + auto data = ColumnString::create(); + data->insert_default(); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(1); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1) + .is()); +} + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeAcceptsSuccessfulCast) { + auto data = ColumnString::create(); + data->insert_data("ok", 2); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(0); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1).ok()); +} + +} // namespace doris::format::jdbc diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index 683a47dee299cc..e8d9aec40af34a 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -173,14 +173,21 @@ struct ReadResult { bool eof = false; size_t second_rows = 0; bool second_eof = false; + size_t document_buffer_size = 0; std::vector schema; }; ReadResult read_once(const std::string& file_name, const std::string& content, TFileScanRangeParams params, const std::vector& slots, - const std::vector& requested_local_ids, bool read_twice = false) { + const std::vector& requested_local_ids, bool read_twice = false, + bool is_hive_table = false) { const auto file_path = write_json_file(file_name, content); auto range = file_range(file_path); + if (is_hive_table) { + TTableFormatFileDesc table_format; + table_format.__set_table_format_type("hive"); + range.__set_table_format_params(std::move(table_format)); + } auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -210,6 +217,7 @@ ReadResult read_once(const std::string& file_name, const std::string& content, result.block = make_block(result.schema, requested_local_ids); result.status = reader.get_block(&result.block, &result.rows, &result.eof); + result.document_buffer_size = reader.TEST_document_buffer_size(); if (result.status.ok() && read_twice) { auto eof_block = make_block(result.schema, requested_local_ids); result.second_status = @@ -318,6 +326,20 @@ TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) { ASSERT_TRUE(result.second_status.ok()) << result.second_status.to_string(); EXPECT_EQ(result.second_rows, 0); EXPECT_TRUE(result.second_eof); + EXPECT_EQ(result.document_buffer_size, 0); +} + +TEST(JsonReaderTest, HiveColumnLookupIsCaseInsensitiveWithoutNormalizedKeys) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("hive_case.jsonl", + R"({"ID":7,"NaMe":"alice"})" + "\n", + json_scan_params(), slots, {0, 1}, false, true); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 7); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "alice"); } TEST(JsonReaderTest, ReadsSingleDocumentOuterArray) { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 1cf6fdaade7cc4..599c930f312856 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -570,7 +570,8 @@ Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physi } Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, - tparquet::CompressionCodec::type codec, bool preload_page_cache = false) { + tparquet::CompressionCodec::type codec, bool preload_page_cache = false, + tparquet::Type::type physical_type = tparquet::Type::INT32) { std::vector bytes; ThriftSerializer serializer(/*compact=*/true, 128); RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); @@ -592,7 +593,7 @@ Status load_scripted_page(tparquet::PageHeader header, const std::vector(); + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BYTE_ARRAY) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } field.repetition_level = 0; field.definition_level = 0; ParquetPageReadContext context(preload_page_cache, page_cache_file_key); @@ -1279,6 +1284,80 @@ TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutio verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); } +TEST(ParquetV2NativeDecoderTest, DictionaryProbeMaterializesTypedValuesOnlyOnce) { + const std::array dictionary {10, 20}; + std::vector dictionary_payload(sizeof(dictionary)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values(dictionary.size()); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + const auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (const uint32_t id : {1U, 0U, 0U, 0U, 0U, 0U, 0U, 0U}) { + encoder.Put(id); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 1; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(2); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(data_page_offset); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + auto file = std::make_shared(bytes); + const auto row_ranges = ::doris::RowRanges::create_single(2); + ScalarColumnReader reader(row_ranges, 2, chunk, nullptr, nullptr, nullptr); + ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + auto dictionary_result = reader.dictionary_values(field.data_type); + ASSERT_TRUE(dictionary_result.has_value()) << dictionary_result.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 2, false).ok()); + ColumnPtr ids = ColumnInt32::create(); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader.read_column_data(ids, field.data_type, nullptr, filter, 2, &rows, &eof, true) + .ok()); + ASSERT_EQ(rows, 2); + auto matched_values = reader.materialize_dictionary_values( + &assert_cast(*ids), field.data_type); + ASSERT_TRUE(matched_values.has_value()) << matched_values.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + EXPECT_EQ(assert_cast(**matched_values).get_data(), + (ColumnInt32::Container {20, 10})); +} + TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { const std::array dictionary_values {10, 20}; auto dictionary = make_unique_buffer(sizeof(dictionary_values)); @@ -2009,6 +2088,24 @@ TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { } } +TEST(ParquetV2NativeDecoderTest, DeltaBinaryPackedRejectsNonIntegralBlockGeometry) { + std::vector encoded(32); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, 3200); + cursor = encode_varint32(cursor, 33); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + Slice slice(encoded.data(), encoded.size()); + EXPECT_FALSE(decoder->set_data(&slice).ok()); +} + TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { const ParquetSelection selection { .total_values = 3, @@ -2699,6 +2796,73 @@ TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); } +TEST(ParquetV2NativeDecoderTest, DecompressionScratchStaysActiveUntilPageExhaustion) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + auto compressed_page = [&](uint32_t value_count) { + std::vector encoded_levels(8); + uint8_t* level_end = encode_varint32(encoded_levels.data(), value_count << 1); + *level_end++ = 1; + encoded_levels.resize(level_end - encoded_levels.data()); + std::vector payload(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + payload.resize(payload.size() + static_cast(value_count) * sizeof(int32_t)); + + faststring compressed; + DORIS_CHECK(codec->compress(Slice(payload.data(), payload.size()), &compressed).ok()); + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value_count); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, std::vector(compressed.data(), + compressed.data() + compressed.size())); + }; + + constexpr uint32_t LARGE_VALUE_COUNT = 1U << 20; + auto bytes = compressed_page(LARGE_VALUE_COUNT); + const auto ordinary_page = compressed_page(1); + bytes.insert(bytes.end(), ordinary_page.begin(), ordinary_page.end()); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(static_cast(LARGE_VALUE_COUNT) + 1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, + static_cast(LARGE_VALUE_COUNT) + 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + ASSERT_GT(reader.active_decoder_scratch_bytes(), 1UL << 20); + ASSERT_TRUE(reader.skip_values(LARGE_VALUE_COUNT).ok()); + ASSERT_TRUE(reader.next_page().ok()); + ASSERT_TRUE(reader.parse_page_header().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + EXPECT_LT(reader.active_decoder_scratch_bytes(), 64UL << 10); + const size_t retained_while_active = reader.retained_decoder_scratch_bytes(); + ASSERT_GT(retained_while_active, 1UL << 20); + reader.release_decoder_scratch(64UL << 10); + EXPECT_EQ(reader.retained_decoder_scratch_bytes(), retained_while_active); + + ASSERT_TRUE(reader.skip_values(1).ok()); + ASSERT_TRUE(reader.next_page().ok()); + // A release requested while a decoder points into the buffer must execute at the next safe + // page boundary; otherwise periodic probes can never age out a previous large-page capacity. + EXPECT_LE(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { const std::vector values {std::string(4096, 'x'), "shared-prefix-a", "shared-prefix-b", "shared-prefix-c"}; @@ -3386,6 +3550,215 @@ TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndL .is()); } +TEST(ParquetV2NativeDecoderTest, EmptyDictionaryRejectsDeclaredPayloadBeforeAllocation) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(0); + EXPECT_TRUE(validate_dictionary_page_size(header).is()); +} + +TEST(ParquetV2NativeDecoderTest, NonemptyFixedWidthDictionaryRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(validate_dictionary_page_size(header, sizeof(int32_t)).is()); + + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, RequiredFixedWidthPageRejectsImpossibleExtentBeforeAllocation) { + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(encoding); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0) + .is()); + header.__set_uncompressed_page_size(sizeof(int32_t)); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, OptionalV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 1}; + + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(levels.size()); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + } +} + +TEST(ParquetV2NativeDecoderTest, RepeatedV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 0, 2, 1}; + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(2); + header.data_page_header_v2.__set_definition_levels_byte_length(2); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDataPagePreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::BYTE_ARRAY); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY, true, + tparquet::Type::BYTE_ARRAY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDictionaryPreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const Slice payload(compressed.data(), compressed.size()); + + EXPECT_TRUE( + validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, value.size()) + .ok()); + EXPECT_TRUE(validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, 8 << 20) + .is()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, false, tparquet::Type::BYTE_ARRAY) + .is()); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, true, tparquet::Type::BYTE_ARRAY) + .is()); +} + TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { tparquet::PageHeader header; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index cc1d28238cb7e1..9bdb0c7562f0b6 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -928,6 +928,14 @@ TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); } +TEST(ParquetScanDeleteConjunctTest, RejectsInputColumnAsEphemeralResult) { + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 0, 2) + .is()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 2, 3).ok()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 3, 3) + .is()); +} + TEST(ParquetScanAdaptivePredicateTest, ThrowingNestedFunctionDisablesSelectedRowReordering) { using format::parquet::detail::AdaptivePredicateStats; std::unordered_map first_batch_stats; @@ -1630,6 +1638,48 @@ TEST_F(ParquetScanTest, ProjectedDeltaBinaryPackedUsesFixedWidthFilterAndProject EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); } +TEST_F(ParquetScanTest, PlainPredicateDirectPathCrossesScratchProbeCadence) { + constexpr int64_t ROWS = 32; + std::vector ids(ROWS); + std::vector scores(ROWS); + std::iota(ids.begin(), ids.end(), 1); + std::iota(scores.begin(), scores.end(), 10); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores)}); + write_table(_file_path, table, ROWS, false, false, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector file_schema; + ASSERT_TRUE(reader->get_schema(&file_schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(file_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS); + // Direct predicate evaluation must survive multiple 16-batch scratch probes; otherwise this + // path can retain a previous outlier for the whole row group without ever aging its capacity. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), ROWS); +} + TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { write_uint32_pair_parquet_file(_file_path); RuntimeProfile profile("profile"); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index d116f84fbaf8a0..74781c0e532b31 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -26,6 +26,7 @@ #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" #include "format_v2/parquet/native_schema_desc.h" @@ -606,4 +607,115 @@ TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingM } } +TEST(ParquetSchemaTest, NativeProjectionUsesResolvedIdentityBeforeCaseInsensitiveFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement group; + group.__set_name("s"); + group.__set_num_children(3); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + tparquet::SchemaElement upper; + upper.__set_name("Value"); + upper.__set_type(tparquet::Type::INT32); + upper.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + upper.__set_field_id(1); + tparquet::SchemaElement lower = upper; + lower.__set_name("value"); + lower.__set_field_id(2); + tparquet::SchemaElement other = upper; + other.__set_name("other"); + other.__set_field_id(3); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, group, upper, lower, other}).ok()); + descriptor.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + + const auto int_type = make_nullable(std::make_shared()); + std::shared_ptr mapping; + auto other_only = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"other"})); + ASSERT_TRUE(build_native_schema_node(other_only, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("other")); + + auto exact_case = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"Value"})); + ASSERT_TRUE(build_native_schema_node(exact_case, *fields[0], &mapping).ok()); + EXPECT_EQ(mapping->file_child_name("Value"), "Value"); + + auto ambiguous_fallback = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"VALUE"})); + const auto status = build_native_schema_node(ambiguous_fallback, *fields[0], &mapping); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetSchemaTest, NativeSetMapKeyValueWrapperRemainsSingleListElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement set; + set.__set_name("tags"); + set.__set_num_children(1); + set.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + set.__set_converted_type(tparquet::ConvertedType::MAP); + tparquet::SchemaElement wrapper; + wrapper.__set_name("key_value"); + wrapper.__set_num_children(1); + wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + wrapper.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, set, wrapper, key}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_INT); +} + +TEST(ParquetSchemaTest, NativeListPreservesLegacyMapKeyValueElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement list; + list.__set_name("entries"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_converted_type(tparquet::ConvertedType::LIST); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_num_children(1); + element.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + element.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement map; + map.__set_name("map"); + map.__set_num_children(2); + map.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::BYTE_ARRAY); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, list, element, map, key, value}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_MAP); + ASSERT_EQ(column->children[0].children.size(), 2); + EXPECT_EQ(remove_nullable(column->children[0].children[0].data_type)->get_primitive_type(), + TYPE_INT); + EXPECT_EQ(remove_nullable(column->children[0].children[1].data_type)->get_primitive_type(), + TYPE_STRING); +} + } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index b8dd97e57d585f..f5bd70292e51a3 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -409,6 +409,13 @@ TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); } +TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(2, 7); + EXPECT_EQ(reader.condition_cache_hit_count(), 9); +} + TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 2434d748dfa109..77466439c6be78 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -840,6 +840,13 @@ TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitc EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); } +TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(3, 5); + EXPECT_EQ(reader.condition_cache_hit_count(), 8); +} + TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp index b17f82f505c2c9..a8affbde3117a3 100644 --- a/be/test/format_v2/table/remote_doris_reader_test.cpp +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -18,11 +18,17 @@ #include "format_v2/table/remote_doris_reader.h" #include +#include #include #include +#include +#include +#include #include +#include #include +#include #include #include #include @@ -48,6 +54,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "testutil/desc_tbl_builder.h" +#include "testutil/mock/mock_runtime_state.h" namespace doris::format::remote_doris { namespace { @@ -95,6 +102,122 @@ TFileRangeDesc remote_doris_range() { return range; } +class BlockingRecordBatchReader final : public arrow::RecordBatchReader { +public: + std::shared_ptr schema() const override { + return arrow::schema({arrow::field("id", arrow::int32())}); + } + + arrow::Status ReadNext(std::shared_ptr* batch) override { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + _cv.wait(lock, [this] { return _released; }); + *batch = nullptr; + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + } + +private: + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +class BlockingFlightServer final : public arrow::flight::FlightServerBase { +public: + enum class Mode { DO_GET, NEXT }; + + explicit BlockingFlightServer(Mode mode) + : _mode(mode), _batch_reader(std::make_shared()) {} + + arrow::Status start() { + auto location = arrow::flight::Location::ForGrpcTcp("localhost", 0); + if (!location.ok()) { + return location.status(); + } + // FlightServerBase::Init starts serving immediately; the blocking Serve lifecycle wrapper + // is unnecessary for an in-process unit test. + return Init(arrow::flight::FlightServerOptions(*location)); + } + + ~BlockingFlightServer() override { + release(); + static_cast(Shutdown()); + } + + arrow::Status DoGet(const arrow::flight::ServerCallContext& context, + const arrow::flight::Ticket&, + std::unique_ptr* stream) override { + if (_mode == Mode::DO_GET) { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + while (!_released && !context.is_cancelled()) { + _cv.wait_for(lock, std::chrono::milliseconds(5)); + } + if (context.is_cancelled()) { + return arrow::Status::Cancelled("client cancelled blocked DoGet"); + } + } + *stream = std::make_unique(_batch_reader); + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + if (_mode == Mode::NEXT) { + return _batch_reader->wait_until_entered(timeout); + } + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + _batch_reader->release(); + } + +private: + Mode _mode; + std::shared_ptr _batch_reader; + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) { + auto range = remote_doris_range(); + auto& params = range.table_format_params.remote_doris_params; + params.__set_location_uri("grpc://localhost:" + std::to_string(server.port())); + arrow::flight::Ticket ticket {.ticket = "ticket"}; + params.__set_ticket(ticket.SerializeToString().ValueOrDie()); + return range; +} + std::vector remote_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { DescriptorTblBuilder builder(pool); builder.declare_tuple() << std::make_tuple(std::make_shared(), std::string("id")) @@ -196,6 +319,16 @@ std::unique_ptr create_reader( std::move(factory)); } +std::unique_ptr create_flight_reader( + RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& slots, std::shared_ptr io_ctx) { + auto system_properties = std::make_shared(); + auto file_description = std::make_unique(); + file_description->path = "/dummyPath"; + return std::make_unique(system_properties, file_description, + std::move(io_ctx), profile, range, slots); +} + Block make_request_block(const std::vector& schema, const std::vector& local_ids) { Block block; @@ -467,4 +600,140 @@ TEST(RemoteDorisV2ReaderTest, RejectsInvalidRemoteDorisRange) { EXPECT_FALSE(reader->init(&state).ok()); } +TEST(RemoteDorisV2ReaderTest, RuntimeCancellationInterruptsBlockedFlightDoGet) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_doget_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + const bool entered = server.wait_until_entered(std::chrono::seconds(2)); + if (!entered && + open_result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { + FAIL() << "Flight DoGet failed before reaching the server: " << open_result.get(); + } + ASSERT_TRUE(entered); + state.cancel(Status::Cancelled("cancel blocked Flight DoGet")); + const bool interrupted = + open_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + EXPECT_FALSE(open_result.get().ok()); +} + +TEST(RemoteDorisV2ReaderTest, BlockedDoGetRetainsQueryResourcesUntilWorkerExits) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + auto state = std::make_unique(); + std::weak_ptr resource_ctx = state->get_query_ctx()->resource_ctx(); + RuntimeProfile profile("remote_doris_v2_doget_resource_context_test"); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, + std::make_shared()); + ASSERT_TRUE(reader->init(state.get()).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + state->cancel(Status::Cancelled("cancel blocked Flight DoGet")); + ASSERT_EQ(open_result.wait_for(std::chrono::milliseconds(750)), std::future_status::ready); + EXPECT_FALSE(open_result.get().ok()); + + reader.reset(); + state.reset(); + // A detached DoGet still owns query-scoped Arrow objects, so it must retain the matching + // resource context until those objects are released on the worker. + EXPECT_FALSE(resource_ctx.expired()); + server.release(); + for (int retries = 0; retries < 100 && !resource_ctx.expired(); ++retries) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_TRUE(resource_ctx.expired()); +} + +TEST(RemoteDorisV2ReaderTest, ScannerStopInterruptsBlockedFlightNext) { + BlockingFlightServer server(BlockingFlightServer::Mode::NEXT); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_next_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto open_status = reader->open(std::move(request)); + ASSERT_TRUE(open_status.ok()) << open_status; + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + auto next_result = + std::async(std::launch::async, [&] { return reader->get_block(&block, &rows, &eof); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + io_ctx->should_stop = true; + const bool interrupted = + next_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + const auto next_status = next_result.get(); + EXPECT_TRUE(!next_status.ok() || (rows == 0 && eof)); + server.release(); +} + +TEST(RemoteDorisV2ReaderTest, CancellationStopsBeforeFlightNext) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_cancel_test"); + auto close_count = std::make_shared(0); + auto io_ctx = std::make_shared(); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id"})}, + close_count, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + io_ctx->should_stop = true; + auto block = make_request_block(schema, {0}); + size_t rows = 99; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + EXPECT_EQ(*close_count, 1); +} + } // namespace doris::format::remote_doris From ccc77ccc02df0f29ebc5d4233c526003f2ccbb39 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 23:33:30 +0800 Subject: [PATCH 23/24] [fix](branch-4.1) Adapt external scan compatibility --- be/src/exec/scan/file_scanner_v2.cpp | 17 ++++ be/src/exec/scan/file_scanner_v2.h | 1 + be/src/exprs/vruntimefilter_wrapper.cpp | 12 +++ be/src/exprs/vruntimefilter_wrapper.h | 1 + be/test/exec/scan/file_scanner_v2_test.cpp | 18 ++++ .../apache/doris/jdbc/BaseJdbcExecutor.java | 18 +++- .../doris/jdbc/BaseJdbcExecutorTest.java | 89 +++++++++++++++++++ 7 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 2ebcf65b2ce944..a058aac64cef8f 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -216,6 +216,18 @@ Status rewrite_slot_refs_to_global_index( return Status::OK(); } +Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { + DORIS_CHECK(expr != nullptr); + if (*expr == nullptr) { + return Status::OK(); + } + if (auto* branch_wrapper = dynamic_cast(expr->get()); + branch_wrapper != nullptr) { + *expr = branch_wrapper->to_runtime_filter_expr(); + } + return Status::OK(); +} + } // namespace #ifdef BE_TEST @@ -251,6 +263,10 @@ Status FileScannerV2::TEST_rewrite_slot_refs_to_global_index( return rewrite_slot_refs_to_global_index(expr, slot_id_to_global_index); } +Status FileScannerV2::TEST_adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { + return adapt_runtime_filter_for_table_reader(expr); +} + FileScannerV2::RealtimeCounterDeltas FileScannerV2::TEST_collect_realtime_counter_deltas( const io::FileReaderStats& file_reader_stats, const io::FileCacheStatistics& file_cache_statistics, @@ -828,6 +844,7 @@ Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, VExprSPtr root; RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); + RETURN_IF_ERROR(adapt_runtime_filter_for_table_reader(&root)); conjuncts->push_back(VExprContext::create_shared(std::move(root))); } return Status::OK(); diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index c68cfb1db8b47c..cabbc96baa8400 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -78,6 +78,7 @@ class FileScannerV2 final : public Scanner { static Status TEST_rewrite_slot_refs_to_global_index( VExprSPtr* expr, const std::unordered_map& slot_id_to_global_index); + static Status TEST_adapt_runtime_filter_for_table_reader(VExprSPtr* expr); static RealtimeCounterDeltas TEST_collect_realtime_counter_deltas( const io::FileReaderStats& file_reader_stats, const io::FileCacheStatistics& file_cache_statistics, diff --git a/be/src/exprs/vruntimefilter_wrapper.cpp b/be/src/exprs/vruntimefilter_wrapper.cpp index 370575ba8661a1..94a2b2221def2e 100644 --- a/be/src/exprs/vruntimefilter_wrapper.cpp +++ b/be/src/exprs/vruntimefilter_wrapper.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type.h" #include "core/types.h" #include "exec/common/util.hpp" +#include "exprs/runtime_filter_expr.h" #include "exprs/vslot_ref.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -84,6 +85,17 @@ Status VRuntimeFilterWrapper::clone_node(VExprSPtr* cloned_expr) const { return Status::OK(); } +VExprSPtr VRuntimeFilterWrapper::to_runtime_filter_expr() const { + // FileScannerV2 backports master TableReader code, whose localization boundary recognizes the + // renamed wrapper type; preserve the same implementation and counters across that boundary. + auto runtime_filter = + RuntimeFilterExpr::create_shared(clone_texpr_node(), _impl, _ignore_thredhold, + _null_aware, _filter_id, _sampling_frequency); + runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, + _always_true_filter_rows); + return runtime_filter; +} + Status VRuntimeFilterWrapper::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); diff --git a/be/src/exprs/vruntimefilter_wrapper.h b/be/src/exprs/vruntimefilter_wrapper.h index 1d903832de353b..d6b736eeebee49 100644 --- a/be/src/exprs/vruntimefilter_wrapper.h +++ b/be/src/exprs/vruntimefilter_wrapper.h @@ -83,6 +83,7 @@ class VRuntimeFilterWrapper final : public VExpr { VExprSPtr get_impl() const override { return _impl; } void set_impl(VExprSPtr impl) { _impl = std::move(impl); } + VExprSPtr to_runtime_filter_expr() const; Status clone_node(VExprSPtr* cloned_expr) const override; void attach_profile_counter(std::shared_ptr rf_input_rows, diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 98f086a4b12ec2..3f3d13c98e9bac 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -38,6 +38,7 @@ #include "exec/scan/file_scanner.h" #include "exec/scan/split_source_connector.h" #include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vbloom_predicate.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vliteral.h" @@ -747,6 +748,23 @@ TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) { } } +TEST(FileScannerV2Test, AdaptsBranchRuntimeFilterForMasterTableReader) { + const auto int_type = std::make_shared(); + const auto node = bool_in_pred_node(); + auto impl = VDirectInPredicate::create_shared(node, nullptr); + impl->add_child(slot_ref(11, 2, int_type, "rf_value")); + VExprSPtr expr = VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0.4, false, 7); + + ASSERT_TRUE(FileScannerV2::TEST_adapt_runtime_filter_for_table_reader(&expr).ok()); + const auto* runtime_filter = dynamic_cast(expr.get()); + ASSERT_NE(runtime_filter, nullptr); + ASSERT_NE(runtime_filter->get_impl(), nullptr); + ASSERT_EQ(runtime_filter->get_impl()->get_num_children(), 1); + const auto* child = + assert_cast(runtime_filter->get_impl()->children()[0].get()); + EXPECT_EQ(child->column_id(), 2); +} + TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { const auto bool_type = std::make_shared(); auto unsafe_predicate = std::make_shared(); diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java index efebe6924136db..b4b15c591e27b1 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java @@ -53,6 +53,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; @@ -589,7 +590,7 @@ protected void setValidationQuery(HikariDataSource ds) { protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, String sql) throws SQLException { if (config.getOp() == TJdbcOperation.READ) { - conn.setAutoCommit(false); + disableAutoCommitIfSupported(conn); Preconditions.checkArgument(sql != null, "SQL statement cannot be null for READ operation."); stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); stmt.setFetchSize(config.getBatchSize()); // set fetch size to batch size @@ -601,6 +602,20 @@ protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, } } + static void disableAutoCommitIfSupported(Connection conn) throws SQLException { + // Hikari invalidates its proxy when setAutoCommit throws, so check the transaction + // capability first to keep non-transactional driver connections usable for reads. + if (!conn.getMetaData().supportsTransactions()) { + LOG.info("JDBC driver does not support transactions; continuing in auto-commit read mode"); + return; + } + try { + conn.setAutoCommit(false); + } catch (SQLFeatureNotSupportedException e) { + LOG.info("JDBC driver does not support disabling auto-commit; continuing in read mode"); + } + } + protected abstract Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException; @@ -838,4 +853,3 @@ protected String defaultByteArrayToHexString(byte[] bytes) { return hexString.toString(); } } - diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java new file mode 100644 index 00000000000000..a0d0968ea67979 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.jdbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.concurrent.atomic.AtomicBoolean; + +class BaseJdbcExecutorTest { + @Test + void disableAutoCommitSkipsNonTransactionalDrivers() { + AtomicBoolean setAutoCommitCalled = new AtomicBoolean(); + Connection connection = connectionWithTransactionSupport(false, setAutoCommitCalled, null); + + Assertions.assertDoesNotThrow( + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + Assertions.assertFalse(setAutoCommitCalled.get()); + } + + @Test + void disableAutoCommitAllowsUnsupportedTransactionalDrivers() { + AtomicBoolean setAutoCommitCalled = new AtomicBoolean(); + Connection connection = connectionWithTransactionSupport( + true, setAutoCommitCalled, new SQLFeatureNotSupportedException("unsupported")); + + Assertions.assertDoesNotThrow( + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + Assertions.assertTrue(setAutoCommitCalled.get()); + } + + @Test + void disableAutoCommitPropagatesOtherSqlErrors() { + Connection connection = connectionWithTransactionSupport( + true, new AtomicBoolean(), new SQLException("connection failed")); + + Assertions.assertThrows(SQLException.class, + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + } + + private static Connection connectionWithTransactionSupport( + boolean supportsTransactions, AtomicBoolean setAutoCommitCalled, SQLException exception) { + DatabaseMetaData metadata = (DatabaseMetaData) Proxy.newProxyInstance( + BaseJdbcExecutorTest.class.getClassLoader(), + new Class[] {DatabaseMetaData.class}, + (proxy, method, args) -> { + if (method.getName().equals("supportsTransactions")) { + return supportsTransactions; + } + throw new UnsupportedOperationException(method.getName()); + }); + return (Connection) Proxy.newProxyInstance( + BaseJdbcExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, args) -> { + if (method.getName().equals("getMetaData")) { + return metadata; + } + if (method.getName().equals("setAutoCommit")) { + setAutoCommitCalled.set(true); + if (exception != null) { + throw exception; + } + return null; + } + throw new UnsupportedOperationException(method.getName()); + }); + } +} From 2287c7cb1588a3dab6ead7b4a2c9fbc6f45394c2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 24 Jul 2026 06:26:38 +0800 Subject: [PATCH 24/24] [fix](iceberg) Finalize position delete reader columns --- ...eberg_position_delete_sys_table_reader.cpp | 14 +++++++ ...iceberg_position_delete_sys_table_reader.h | 5 +++ ..._position_delete_sys_table_reader_test.cpp | 39 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp index c01027179dd2fd..0ef726318d9d9a 100644 --- a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -213,6 +213,20 @@ Status IcebergPositionDeleteSysTableReader::get_columns( return Status::OK(); } +Status IcebergPositionDeleteSysTableReader::set_fill_columns( + const std::unordered_map>& + partition_columns, + const std::unordered_map& missing_columns, + const std::unordered_map& partition_value_is_null) { + if (_position_reader == nullptr) { + return Status::OK(); + } + // Native readers finalize their physical read-column plan in set_fill_columns even when all + // maps are empty. Skipping this leaves the nested reader yielding empty non-EOF batches. + return _position_reader->set_fill_columns(partition_columns, missing_columns, + partition_value_is_null); +} + bool IcebergPositionDeleteSysTableReader::count_read_rows() { return _delete_file_kind == DeleteFileKind::POSITION_DELETE; } diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.h b/be/src/format/table/iceberg_position_delete_sys_table_reader.h index ae16486395f97a..c09a879dd0f463 100644 --- a/be/src/format/table/iceberg_position_delete_sys_table_reader.h +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.h @@ -56,6 +56,11 @@ class IcebergPositionDeleteSysTableReader : public GenericReader { Status get_next_block(Block* block, size_t* read_rows, bool* eof) override; Status get_columns(std::unordered_map* name_to_type, std::unordered_set* missing_cols) override; + Status set_fill_columns( + const std::unordered_map>& + partition_columns, + const std::unordered_map& missing_columns, + const std::unordered_map& partition_value_is_null = {}) override; void set_batch_size(size_t batch_size) override { _batch_size = batch_size; } size_t get_batch_size() const override { return _batch_size; } bool count_read_rows() override; diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index e2ddebb983e899..f3c5ef1c386c8f 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -57,6 +57,24 @@ class ProfileTrackingReader final : public GenericReader { void _collect_profile_before_close() override { ++collect_calls; } }; +class FillColumnsTrackingReader final : public GenericReader { +public: + int set_fill_columns_calls = 0; + + Status get_next_block(Block* /*block*/, size_t* /*read_rows*/, bool* /*eof*/) override { + return Status::OK(); + } + + Status set_fill_columns( + const std::unordered_map>& + /*partition_columns*/, + const std::unordered_map& /*missing_columns*/, + const std::unordered_map& /*partition_value_is_null*/) override { + ++set_fill_columns_calls; + return Status::OK(); + } +}; + class RejectAllRowsPredicate final : public VExpr { public: RejectAllRowsPredicate() : VExpr(std::make_shared(), false) {} @@ -224,6 +242,27 @@ TEST(IcebergPositionDeleteSysTableReaderTest, StopsBeforeExpandingDeletionVector EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableReaderTest, FinalizesNestedReaderColumns) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + auto nested_reader = std::make_unique(); + auto* nested_reader_ptr = nested_reader.get(); + reader._position_reader = std::move(nested_reader); + + std::unordered_map> + partition_columns; + std::unordered_map missing_columns; + ASSERT_TRUE(reader.set_fill_columns(partition_columns, missing_columns).ok()); + EXPECT_EQ(1, nested_reader_ptr->set_fill_columns_calls); +} + TEST(IcebergPositionDeleteSysTableReaderTest, ValidatesRangeAndDeleteFileMetadata) { RuntimeState state {TQueryOptions(), TQueryGlobals()}; RuntimeProfile profile("test_profile");